lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject svn commit: r1297785 [10/16] - in /lucene/dev/branches/lucene3795_lsp_spatial_module: ./ dev-tools/eclipse/ dev-tools/maven/ dev-tools/maven/lucene/ dev-tools/maven/lucene/contrib/demo/ dev-tools/maven/lucene/contrib/highlighter/ dev-tools/maven/lucene...
Date Tue, 06 Mar 2012 23:17:23 GMT
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java Tue Mar  6 23:17:08 2012
@@ -19,16 +19,24 @@ package org.apache.lucene.search.suggest
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 
 import org.apache.lucene.search.spell.TermFreqIterator;
 import org.apache.lucene.search.suggest.Lookup;
+import org.apache.lucene.search.suggest.SortedTermFreqIteratorWrapper;
+import org.apache.lucene.search.suggest.fst.Sort.ByteSequencesWriter;
 import org.apache.lucene.store.ByteArrayDataInput;
 import org.apache.lucene.store.ByteArrayDataOutput;
+import org.apache.lucene.store.InputStreamDataInput;
+import org.apache.lucene.store.OutputStreamDataOutput;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.IntsRef;
 import org.apache.lucene.util.UnicodeUtil;
@@ -50,7 +58,7 @@ import org.apache.lucene.util.fst.Util.M
  * Input weights will be cast to a java integer, and any
  * negative, infinite, or NaN values will be rejected.
  * 
- * @see Util#shortestPaths(FST, FST.Arc, int)
+ * @see Util#shortestPaths(FST, FST.Arc, Comparator, int)
  * @lucene.experimental
  */
 public class WFSTCompletionLookup extends Lookup {
@@ -96,74 +104,27 @@ public class WFSTCompletionLookup extend
   
   @Override
   public void build(TermFreqIterator iterator) throws IOException {
-    String prefix = getClass().getSimpleName();
-    File directory = Sort.defaultTempDir();
-    File tempInput = File.createTempFile(prefix, ".input", directory);
-    File tempSorted = File.createTempFile(prefix, ".sorted", directory);
-    
-    Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput);
-    Sort.ByteSequencesReader reader = null;
     BytesRef scratch = new BytesRef();
-    
-    boolean success = false;
-    try {
-      byte [] buffer = new byte [0];
-      ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
-      while (iterator.hasNext()) {
-        String key = iterator.next();
-        UnicodeUtil.UTF16toUTF8(key, 0, key.length(), scratch);
-
-        if (scratch.length + 5 >= buffer.length) {
-          buffer = ArrayUtil.grow(buffer, scratch.length + 5);
-        }
-
-        output.reset(buffer);
-        output.writeBytes(scratch.bytes, scratch.offset, scratch.length);
-        output.writeByte((byte)0); // separator: not used, just for sort order
-        output.writeInt((int)encodeWeight(iterator.freq()));
-        writer.write(buffer, 0, output.getPosition());
-      }
-      writer.close();
-      new Sort().sort(tempInput, tempSorted);
-      reader = new Sort.ByteSequencesReader(tempSorted);
+    TermFreqIterator iter = new WFSTTermFreqIteratorWrapper(iterator,
+        BytesRef.getUTF8SortedAsUnicodeComparator());
+    IntsRef scratchInts = new IntsRef();
+    BytesRef previous = null;
+    PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
+    Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
+    while ((scratch = iter.next()) != null) {
+      long cost = iter.weight();
       
-      PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
-      Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
-      
-      BytesRef previous = null;
-      BytesRef suggestion = new BytesRef();
-      IntsRef scratchInts = new IntsRef();
-      ByteArrayDataInput input = new ByteArrayDataInput();
-      while (reader.read(scratch)) {
-        suggestion.bytes = scratch.bytes;
-        suggestion.offset = scratch.offset;
-        suggestion.length = scratch.length - 5; // int + separator
-
-        input.reset(scratch.bytes);
-        input.skipBytes(suggestion.length + 1); // suggestion + separator
-        long cost = input.readInt();
-   
-        if (previous == null) {
-          previous = new BytesRef();
-        } else if (suggestion.equals(previous)) {
-          continue; // for duplicate suggestions, the best weight is actually added
-        }
-        Util.toIntsRef(suggestion, scratchInts);
-        builder.add(scratchInts, cost);
-        previous.copyBytes(suggestion);
-      }
-      fst = builder.finish();
-      success = true;
-    } finally {
-      if (success) {
-        IOUtils.close(reader, writer);
-      } else {
-        IOUtils.closeWhileHandlingException(reader, writer);
+      if (previous == null) {
+        previous = new BytesRef();
+      } else if (scratch.equals(previous)) {
+        continue; // for duplicate suggestions, the best weight is actually
+                  // added
       }
-      
-      tempInput.delete();
-      tempSorted.delete();
+      Util.toIntsRef(scratch, scratchInts);
+      builder.add(scratchInts, cost);
+      previous.copyBytes(scratch);
     }
+    fst = builder.finish();
   }
 
   @Override
@@ -177,9 +138,29 @@ public class WFSTCompletionLookup extend
     this.fst = FST.read(new File(storeDir, FILENAME), PositiveIntOutputs.getSingleton(true));
     return true;
   }
+  
+  @Override
+  public boolean store(OutputStream output) throws IOException {
+    try {
+      fst.save(new OutputStreamDataOutput(output));
+    } finally {
+      IOUtils.close(output);
+    }
+    return true;
+  }
 
   @Override
-  public List<LookupResult> lookup(String key, boolean onlyMorePopular, int num) {
+  public boolean load(InputStream input) throws IOException {
+    try {
+      this.fst = new FST<Long>(new InputStreamDataInput(input), PositiveIntOutputs.getSingleton(true));
+    } finally {
+      IOUtils.close(input);
+    }
+    return true;
+  }
+
+  @Override
+  public List<LookupResult> lookup(CharSequence key, boolean onlyMorePopular, int num) {
     assert num > 0;
     BytesRef scratch = new BytesRef(key);
     int prefixLength = scratch.length;
@@ -196,27 +177,31 @@ public class WFSTCompletionLookup extend
     }
     
     List<LookupResult> results = new ArrayList<LookupResult>(num);
+    CharsRef spare = new CharsRef();
     if (exactFirst && arc.isFinal()) {
-      results.add(new LookupResult(scratch.utf8ToString(), decodeWeight(prefixOutput + arc.nextFinalOutput)));
+      spare.grow(scratch.length);
+      UnicodeUtil.UTF8toUTF16(scratch, spare);
+      results.add(new LookupResult(spare.toString(), decodeWeight(prefixOutput + arc.nextFinalOutput)));
       if (--num == 0) {
         return results; // that was quick
       }
     }
     
     // complete top-N
-    MinResult completions[] = null;
+    MinResult<Long> completions[] = null;
     try {
-      completions = Util.shortestPaths(fst, arc, num);
+      completions = Util.shortestPaths(fst, arc, weightComparator, num);
     } catch (IOException bogus) { throw new RuntimeException(bogus); }
     
     BytesRef suffix = new BytesRef(8);
-    for (MinResult completion : completions) {
+    for (MinResult<Long> completion : completions) {
       scratch.length = prefixLength;
       // append suffix
       Util.toBytesRef(completion.input, suffix);
       scratch.append(suffix);
-
-      results.add(new LookupResult(scratch.utf8ToString(), decodeWeight(prefixOutput + completion.output)));
+      spare.grow(scratch.length);
+      UnicodeUtil.UTF8toUTF16(scratch, spare);
+      results.add(new LookupResult(spare.toString(), decodeWeight(prefixOutput + completion.output)));
     }
     return results;
   }
@@ -242,17 +227,11 @@ public class WFSTCompletionLookup extend
     return output;
   }
   
-  @Override
-  public boolean add(String key, Object value) {
-    return false; // Not supported.
-  }
-
   /**
    * Returns the weight associated with an input string,
    * or null if it does not exist.
    */
-  @Override
-  public Float get(String key) {
+  public Object get(CharSequence key) {
     Arc<Long> arc = new Arc<Long>();
     Long result = null;
     try {
@@ -261,20 +240,54 @@ public class WFSTCompletionLookup extend
     if (result == null || !arc.isFinal()) {
       return null;
     } else {
-      return decodeWeight(result + arc.nextFinalOutput);
+      return Integer.valueOf(decodeWeight(result + arc.nextFinalOutput));
     }
   }
   
   /** cost -> weight */
-  private static float decodeWeight(long encoded) {
-    return Integer.MAX_VALUE - encoded;
+  private static int decodeWeight(long encoded) {
+    return (int)(Integer.MAX_VALUE - encoded);
   }
   
   /** weight -> cost */
-  private static long encodeWeight(float value) {
-    if (Float.isNaN(value) || Float.isInfinite(value) || value < 0 || value > Integer.MAX_VALUE) {
+  private static int encodeWeight(long value) {
+    if (value < 0 || value > Integer.MAX_VALUE) {
       throw new UnsupportedOperationException("cannot encode value: " + value);
     }
     return Integer.MAX_VALUE - (int)value;
   }
+  
+  private final class WFSTTermFreqIteratorWrapper extends SortedTermFreqIteratorWrapper {
+
+    WFSTTermFreqIteratorWrapper(TermFreqIterator source,
+        Comparator<BytesRef> comparator) throws IOException {
+      super(source, comparator, true);
+    }
+
+    @Override
+    protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, long weight) throws IOException {
+      if (spare.length + 5 >= buffer.length) {
+        buffer = ArrayUtil.grow(buffer, spare.length + 5);
+      }
+      output.reset(buffer);
+      output.writeBytes(spare.bytes, spare.offset, spare.length);
+      output.writeByte((byte)0); // separator: not used, just for sort order
+      output.writeInt(encodeWeight(weight));
+      writer.write(buffer, 0, output.getPosition());
+    }
+    
+    @Override
+    protected long decode(BytesRef scratch, ByteArrayDataInput tmpInput) {
+      tmpInput.reset(scratch.bytes);
+      tmpInput.skipBytes(scratch.length - 4); // suggestion + separator
+      scratch.length -= 5; // sep + long
+      return tmpInput.readInt();
+    }
+  }
+  
+  static final Comparator<Long> weightComparator = new Comparator<Long> () {
+    public int compare(Long left, Long right) {
+      return left.compareTo(right);
+    }  
+  };
 }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java Tue Mar  6 23:17:08 2012
@@ -23,14 +23,19 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.lucene.search.spell.SortedIterator;
 import org.apache.lucene.search.spell.TermFreqIterator;
 import org.apache.lucene.search.suggest.Lookup;
 import org.apache.lucene.search.suggest.UnsortedTermFreqIteratorWrapper;
 import org.apache.lucene.search.suggest.jaspell.JaspellTernarySearchTrie.TSTNode;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.UnicodeUtil;
 
 public class JaspellLookup extends Lookup {
   JaspellTernarySearchTrie trie = new JaspellTernarySearchTrie();
@@ -39,36 +44,39 @@ public class JaspellLookup extends Looku
 
   @Override
   public void build(TermFreqIterator tfit) throws IOException {
-    if (tfit instanceof SortedIterator) {
+    if (tfit.getComparator() != null) {
       // make sure it's unsorted
+      // WTF - this could result in yet another sorted iteration....
       tfit = new UnsortedTermFreqIteratorWrapper(tfit);
     }
     trie = new JaspellTernarySearchTrie();
     trie.setMatchAlmostDiff(editDistance);
-    while (tfit.hasNext()) {
-      String key = tfit.next();
-      float freq = tfit.freq();
-      if (key.length() == 0) {
+    BytesRef spare;
+    final CharsRef charsSpare = new CharsRef();
+
+    while ((spare = tfit.next()) != null) {
+      final long weight = tfit.weight();
+      if (spare.length == 0) {
         continue;
       }
-      trie.put(key, new Float(freq));
+      charsSpare.grow(spare.length);
+      UnicodeUtil.UTF8toUTF16(spare.bytes, spare.offset, spare.length, charsSpare);
+      trie.put(charsSpare.toString(), Long.valueOf(weight));
     }
   }
 
-  @Override
-  public boolean add(String key, Object value) {
+  public boolean add(CharSequence key, Object value) {
     trie.put(key, value);
     // XXX
     return false;
   }
 
-  @Override
-  public Object get(String key) {
+  public Object get(CharSequence key) {
     return trie.get(key);
   }
 
   @Override
-  public List<LookupResult> lookup(String key, boolean onlyMorePopular, int num) {
+  public List<LookupResult> lookup(CharSequence key, boolean onlyMorePopular, int num) {
     List<LookupResult> res = new ArrayList<LookupResult>();
     List<String> list;
     int count = onlyMorePopular ? num * 2 : num;
@@ -85,8 +93,8 @@ public class JaspellLookup extends Looku
     if (onlyMorePopular) {
       LookupPriorityQueue queue = new LookupPriorityQueue(num);
       for (String s : list) {
-        float freq = (Float)trie.get(s);
-        queue.insertWithOverflow(new LookupResult(s, freq));
+        long freq = ((Number)trie.get(s)).longValue();
+        queue.insertWithOverflow(new LookupResult(new CharsRef(s), freq));
       }
       for (LookupResult lr : queue.getResults()) {
         res.add(lr);
@@ -94,8 +102,8 @@ public class JaspellLookup extends Looku
     } else {
       for (int i = 0; i < maxCnt; i++) {
         String s = list.get(i);
-        float freq = (Float)trie.get(s);
-        res.add(new LookupResult(s, freq));
+        long freq = ((Number)trie.get(s)).longValue();
+        res.add(new LookupResult(new CharsRef(s), freq));
       }      
     }
     return res;
@@ -114,22 +122,14 @@ public class JaspellLookup extends Looku
     if (!data.exists() || !data.canRead()) {
       return false;
     }
-    DataInputStream in = new DataInputStream(new FileInputStream(data));
-    TSTNode root = trie.new TSTNode('\0', null);
-    try {
-      readRecursively(in, root);
-      trie.setRoot(root);
-    } finally {
-      in.close();
-    }
-    return true;
+    return load(new FileInputStream(data));
   }
   
   private void readRecursively(DataInputStream in, TSTNode node) throws IOException {
     node.splitchar = in.readChar();
     byte mask = in.readByte();
     if ((mask & HAS_VALUE) != 0) {
-      node.data = new Float(in.readFloat());
+      node.data = Long.valueOf(in.readLong());
     }
     if ((mask & LO_KID) != 0) {
       TSTNode kid = trie.new TSTNode('\0', node);
@@ -153,19 +153,8 @@ public class JaspellLookup extends Looku
     if (!storeDir.exists() || !storeDir.isDirectory() || !storeDir.canWrite()) {
       return false;
     }
-    TSTNode root = trie.getRoot();
-    if (root == null) { // empty tree
-      return false;
-    }
     File data = new File(storeDir, FILENAME);
-    DataOutputStream out = new DataOutputStream(new FileOutputStream(data));
-    try {
-      writeRecursively(out, root);
-      out.flush();
-    } finally {
-      out.close();
-    }
-    return true;
+    return store(new FileOutputStream(data));
   }
   
   private void writeRecursively(DataOutputStream out, TSTNode node) throws IOException {
@@ -180,10 +169,39 @@ public class JaspellLookup extends Looku
     if (node.data != null) mask |= HAS_VALUE;
     out.writeByte(mask);
     if (node.data != null) {
-      out.writeFloat((Float)node.data);
+      out.writeLong(((Number)node.data).longValue());
     }
     writeRecursively(out, node.relatives[TSTNode.LOKID]);
     writeRecursively(out, node.relatives[TSTNode.EQKID]);
     writeRecursively(out, node.relatives[TSTNode.HIKID]);
   }
+
+  @Override
+  public boolean store(OutputStream output) throws IOException {
+    TSTNode root = trie.getRoot();
+    if (root == null) { // empty tree
+      return false;
+    }
+    DataOutputStream out = new DataOutputStream(output);
+    try {
+      writeRecursively(out, root);
+      out.flush();
+    } finally {
+      IOUtils.close(out);
+    }
+    return true;
+  }
+
+  @Override
+  public boolean load(InputStream input) throws IOException {
+    DataInputStream in = new DataInputStream(input);
+    TSTNode root = trie.new TSTNode('\0', null);
+    try {
+      readRecursively(in, root);
+      trie.setRoot(root);
+    } finally {
+      IOUtils.close(in);
+    }
+    return true;
+  }
 }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java Tue Mar  6 23:17:08 2012
@@ -368,8 +368,8 @@ public class JaspellTernarySearchTrie {
    *          A <code>String</code> index.
    *@return The object retrieved from the Ternary Search Trie.
    */
-  public Object get(String key) {
-    TSTNode node = getNode(key.trim().toLowerCase());
+  public Object get(CharSequence key) {
+    TSTNode node = getNode(key);
     if (node == null) {
       return null;
     }
@@ -435,7 +435,7 @@ public class JaspellTernarySearchTrie {
    *@return The node object indexed by key. This object is an instance of an
    *         inner class named <code>TernarySearchTrie.TSTNode</code>.
    */
-  public TSTNode getNode(String key) {
+  public TSTNode getNode(CharSequence key) {
     return getNode(key, rootNode);
   }
 
@@ -443,15 +443,14 @@ public class JaspellTernarySearchTrie {
    * Returns the node indexed by key, or <code>null</code> if that node doesn't
    * exist. The search begins at root node.
    * 
-   *@param key2
+   *@param key
    *          A <code>String</code> that indexes the node that is returned.
    *@param startNode
    *          The top node defining the subtrie to be searched.
    *@return The node object indexed by key. This object is an instance of an
    *         inner class named <code>TernarySearchTrie.TSTNode</code>.
    */
-  protected TSTNode getNode(String key2, TSTNode startNode) {
-    String key = key2.trim().toLowerCase();
+  protected TSTNode getNode(CharSequence key, TSTNode startNode) {
     if (key == null || startNode == null || key.length() == 0) {
       return null;
     }
@@ -490,7 +489,7 @@ public class JaspellTernarySearchTrie {
    *@exception IllegalArgumentException
    *              If the key is an empty <code>String</code>.
    */
-  protected TSTNode getOrCreateNode(String key) throws NullPointerException,
+  protected TSTNode getOrCreateNode(CharSequence key) throws NullPointerException,
           IllegalArgumentException {
     if (key == null) {
       throw new NullPointerException(
@@ -568,7 +567,7 @@ public class JaspellTernarySearchTrie {
    *          The maximum number of values returned by this method.
    *@return A <code>List</code> with the results
    */
-  public List<String> matchAlmost(String key, int numReturnValues) {
+  public List<String> matchAlmost(CharSequence key, int numReturnValues) {
     return matchAlmostRecursion(rootNode, 0, matchAlmostDiff, key,
             ((numReturnValues < 0) ? -1 : numReturnValues), new Vector<String>(), false);
   }
@@ -598,7 +597,7 @@ public class JaspellTernarySearchTrie {
    *@return A <code>List</code> with the results.
    */
   private List<String> matchAlmostRecursion(TSTNode currentNode, int charIndex,
-          int d, String matchAlmostKey, int matchAlmostNumReturnValues,
+          int d, CharSequence matchAlmostKey, int matchAlmostNumReturnValues,
           List<String> matchAlmostResult2, boolean upTo) {
     if ((currentNode == null)
             || (matchAlmostNumReturnValues != -1 && matchAlmostResult2.size() >= matchAlmostNumReturnValues)
@@ -658,7 +657,7 @@ public class JaspellTernarySearchTrie {
    *          The maximum number of values returned from this method.
    *@return A <code>List</code> with the results
    */
-  public List<String> matchPrefix(String prefix, int numReturnValues) {
+  public List<String> matchPrefix(CharSequence prefix, int numReturnValues) {
     Vector<String> sortKeysResult = new Vector<String>();
     TSTNode startNode = getNode(prefix);
     if (startNode == null) {
@@ -722,8 +721,8 @@ public class JaspellTernarySearchTrie {
    *@param value
    *          The object to be stored in the Trie.
    */
-  public void put(String key, Object value) {
-    getOrCreateNode(key.trim().toLowerCase()).data = value;
+  public void put(CharSequence key, Object value) {
+    getOrCreateNode(key).data = value;
   }
 
   /**

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTAutocomplete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTAutocomplete.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTAutocomplete.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTAutocomplete.java Tue Mar  6 23:17:08 2012
@@ -57,7 +57,7 @@ public class TSTAutocomplete {
    *          index of character in key to be inserted currently.
    * @return currentNode The new reference to root node of TST
    */
-  public TernaryTreeNode insert(TernaryTreeNode currentNode, String s,
+  public TernaryTreeNode insert(TernaryTreeNode currentNode, CharSequence s,
           Object val, int x) {
     if (s == null || s.length() <= x) {
       return currentNode;
@@ -69,7 +69,7 @@ public class TSTAutocomplete {
       if (x < s.length() - 1) {
         currentNode.eqKid = insert(currentNode.eqKid, s, val, x + 1);
       } else {
-        currentNode.token = s;
+        currentNode.token = s.toString();
         currentNode.val = val;
         return currentNode;
       }
@@ -79,7 +79,7 @@ public class TSTAutocomplete {
       if (x < s.length() - 1) {
         currentNode.eqKid = insert(currentNode.eqKid, s, val, x + 1);
       } else {
-        currentNode.token = s;
+        currentNode.token = s.toString();
         currentNode.val = val;
         return currentNode;
       }
@@ -104,7 +104,7 @@ public class TSTAutocomplete {
    * @return suggest list of auto-completed keys for the given prefix query.
    */
   public ArrayList<TernaryTreeNode> prefixCompletion(TernaryTreeNode root,
-          String s, int x) {
+          CharSequence s, int x) {
 
     TernaryTreeNode p = root;
     ArrayList<TernaryTreeNode> suggest = new ArrayList<TernaryTreeNode>();

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java Tue Mar  6 23:17:08 2012
@@ -23,13 +23,18 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.lucene.search.suggest.Lookup;
 import org.apache.lucene.search.suggest.SortedTermFreqIteratorWrapper;
-import org.apache.lucene.search.spell.SortedIterator;
 import org.apache.lucene.search.spell.TermFreqIterator;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.UnicodeUtil;
 
 public class TSTLookup extends Lookup {
   TernaryTreeNode root = new TernaryTreeNode();
@@ -39,43 +44,58 @@ public class TSTLookup extends Lookup {
   public void build(TermFreqIterator tfit) throws IOException {
     root = new TernaryTreeNode();
     // buffer first
-    if (!(tfit instanceof SortedIterator)) {
-      // make sure it's sorted
-      tfit = new SortedTermFreqIteratorWrapper(tfit);
+    if (tfit.getComparator() != BytesRef.getUTF8SortedAsUTF16Comparator()) {
+      // make sure it's sorted and the comparator uses UTF16 sort order
+      tfit = new SortedTermFreqIteratorWrapper(tfit, BytesRef.getUTF8SortedAsUTF16Comparator());
     }
 
     ArrayList<String> tokens = new ArrayList<String>();
-    ArrayList<Float> vals = new ArrayList<Float>();
-    while (tfit.hasNext()) {
-      tokens.add(tfit.next());
-      vals.add(new Float(tfit.freq()));
+    ArrayList<Number> vals = new ArrayList<Number>();
+    BytesRef spare;
+    CharsRef charsSpare = new CharsRef();
+    while ((spare = tfit.next()) != null) {
+      charsSpare.grow(spare.length);
+      UnicodeUtil.UTF8toUTF16(spare.bytes, spare.offset, spare.length, charsSpare);
+      tokens.add(charsSpare.toString());
+      vals.add(Long.valueOf(tfit.weight()));
     }
     autocomplete.balancedTree(tokens.toArray(), vals.toArray(), 0, tokens.size() - 1, root);
   }
 
-  @Override
-  public boolean add(String key, Object value) {
+  public boolean add(CharSequence key, Object value) {
     autocomplete.insert(root, key, value, 0);
     // XXX we don't know if a new node was created
     return true;
   }
 
-  @Override
-  public Object get(String key) {
+  public Object get(CharSequence key) {
     List<TernaryTreeNode> list = autocomplete.prefixCompletion(root, key, 0);
     if (list == null || list.isEmpty()) {
       return null;
     }
     for (TernaryTreeNode n : list) {
-      if (n.token.equals(key)) {
+      if (charSeqEquals(n.token, key)) {
         return n.val;
       }
     }
     return null;
   }
+  
+  private static boolean charSeqEquals(CharSequence left, CharSequence right) {
+    int len = left.length();
+    if (len != right.length()) {
+      return false;
+    }
+    for (int i = 0; i < len; i++) {
+      if (left.charAt(i) != right.charAt(i)) {
+        return false;
+      }
+    }
+    return true;
+  }
 
   @Override
-  public List<LookupResult> lookup(String key, boolean onlyMorePopular, int num) {
+  public List<LookupResult> lookup(CharSequence key, boolean onlyMorePopular, int num) {
     List<TernaryTreeNode> list = autocomplete.prefixCompletion(root, key, 0);
     List<LookupResult> res = new ArrayList<LookupResult>();
     if (list == null || list.size() == 0) {
@@ -85,7 +105,7 @@ public class TSTLookup extends Lookup {
     if (onlyMorePopular) {
       LookupPriorityQueue queue = new LookupPriorityQueue(num);
       for (TernaryTreeNode ttn : list) {
-        queue.insertWithOverflow(new LookupResult(ttn.token, (Float)ttn.val));
+        queue.insertWithOverflow(new LookupResult(ttn.token, ((Number)ttn.val).longValue()));
       }
       for (LookupResult lr : queue.getResults()) {
         res.add(lr);
@@ -93,7 +113,7 @@ public class TSTLookup extends Lookup {
     } else {
       for (int i = 0; i < maxCnt; i++) {
         TernaryTreeNode ttn = list.get(i);
-        res.add(new LookupResult(ttn.token, (Float)ttn.val));
+        res.add(new LookupResult(ttn.token, ((Number)ttn.val).longValue()));
       }
     }
     return res;
@@ -113,14 +133,7 @@ public class TSTLookup extends Lookup {
     if (!data.exists() || !data.canRead()) {
       return false;
     }
-    DataInputStream in = new DataInputStream(new FileInputStream(data));
-    root = new TernaryTreeNode();
-    try {
-      readRecursively(in, root);
-    } finally {
-      in.close();
-    }
-    return true;
+    return load(new FileInputStream(data));
   }
   
   // pre-order traversal
@@ -131,7 +144,7 @@ public class TSTLookup extends Lookup {
       node.token = in.readUTF();
     }
     if ((mask & HAS_VALUE) != 0) {
-      node.val = new Float(in.readFloat());
+      node.val = Long.valueOf(in.readLong());
     }
     if ((mask & LO_KID) != 0) {
       node.loKid = new TernaryTreeNode();
@@ -153,14 +166,7 @@ public class TSTLookup extends Lookup {
       return false;
     }
     File data = new File(storeDir, FILENAME);
-    DataOutputStream out = new DataOutputStream(new FileOutputStream(data));
-    try {
-      writeRecursively(out, root);
-      out.flush();
-    } finally {
-      out.close();
-    }
-    return true;
+    return store(new FileOutputStream(data));
   }
   
   // pre-order traversal
@@ -176,7 +182,7 @@ public class TSTLookup extends Lookup {
     if (node.val != null) mask |= HAS_VALUE;
     out.writeByte(mask);
     if (node.token != null) out.writeUTF(node.token);
-    if (node.val != null) out.writeFloat((Float)node.val);
+    if (node.val != null) out.writeLong(((Number)node.val).longValue());
     // recurse and write kids
     if (node.loKid != null) {
       writeRecursively(out, node.loKid);
@@ -188,4 +194,28 @@ public class TSTLookup extends Lookup {
       writeRecursively(out, node.hiKid);
     }
   }
+
+  @Override
+  public synchronized boolean store(OutputStream output) throws IOException {
+    DataOutputStream out = new DataOutputStream(output);
+    try {
+      writeRecursively(out, root);
+      out.flush();
+    } finally {
+      IOUtils.close(output);
+    }
+    return true;
+  }
+
+  @Override
+  public synchronized boolean load(InputStream input) throws IOException {
+    DataInputStream in = new DataInputStream(input);
+    root = new TernaryTreeNode();
+    try {
+      readRecursively(in, root);
+    } finally {
+      IOUtils.close(in);
+    }
+    return true;
+  }
 }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java Tue Mar  6 23:17:08 2012
@@ -18,15 +18,17 @@ package org.apache.lucene.search.spell;
  */
 
 import java.io.IOException;
-import java.util.Iterator;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
 import org.apache.lucene.util.LuceneTestCase;
 
 /**
@@ -40,7 +42,8 @@ public class TestLuceneDictionary extend
 
   private IndexReader indexReader = null;
   private LuceneDictionary ld;
-  private Iterator<String> it;
+  private BytesRefIterator it;
+  private BytesRef spare = new BytesRef();
 
   @Override
   public void setUp() throws Exception {
@@ -84,13 +87,12 @@ public class TestLuceneDictionary extend
   
   public void testFieldNonExistent() throws IOException {
     try {
-      indexReader = IndexReader.open(store);
+      indexReader = DirectoryReader.open(store);
 
       ld = new LuceneDictionary(indexReader, "nonexistent_field");
       it = ld.getWordsIterator();
 
-      assertFalse("More elements than expected", it.hasNext());
-      assertTrue("Nonexistent element is really null", it.next() == null);
+      assertNull("More elements than expected", spare = it.next());
     } finally {
       if  (indexReader != null) { indexReader.close(); }
     }
@@ -98,15 +100,13 @@ public class TestLuceneDictionary extend
 
   public void testFieldAaa() throws IOException {
     try {
-      indexReader = IndexReader.open(store);
+      indexReader = DirectoryReader.open(store);
 
       ld = new LuceneDictionary(indexReader, "aaa");
       it = ld.getWordsIterator();
-
-      assertTrue("First element doesn't exist.", it.hasNext());
-      assertTrue("First element isn't correct", it.next().equals("foo"));
-      assertFalse("More elements than expected", it.hasNext());
-      assertTrue("Nonexistent element is really null", it.next() == null);
+      assertNotNull("First element doesn't exist.", spare = it.next());
+      assertTrue("First element isn't correct", spare.utf8ToString().equals("foo"));
+      assertNull("More elements than expected", it.next());
     } finally {
       if  (indexReader != null) { indexReader.close(); }
     }
@@ -114,24 +114,22 @@ public class TestLuceneDictionary extend
 
   public void testFieldContents_1() throws IOException {
     try {
-      indexReader = IndexReader.open(store);
+      indexReader = DirectoryReader.open(store);
 
       ld = new LuceneDictionary(indexReader, "contents");
       it = ld.getWordsIterator();
 
-      assertTrue("First element doesn't exist.", it.hasNext());
-      assertTrue("First element isn't correct", it.next().equals("Jerry"));
-      assertTrue("Second element doesn't exist.", it.hasNext());
-      assertTrue("Second element isn't correct", it.next().equals("Tom"));
-      assertFalse("More elements than expected", it.hasNext());
-      assertTrue("Nonexistent element is really null", it.next() == null);
+      assertNotNull("First element doesn't exist.", spare = it.next());
+      assertTrue("First element isn't correct", spare.utf8ToString().equals("Jerry"));
+      assertNotNull("Second element doesn't exist.", spare = it.next());
+      assertTrue("Second element isn't correct", spare.utf8ToString().equals("Tom"));
+      assertNull("More elements than expected", it.next());
 
       ld = new LuceneDictionary(indexReader, "contents");
       it = ld.getWordsIterator();
 
       int counter = 2;
-      while (it.hasNext()) {
-        it.next();
+      while (it.next() != null) {
         counter--;
       }
 
@@ -144,30 +142,15 @@ public class TestLuceneDictionary extend
 
   public void testFieldContents_2() throws IOException {
     try {
-      indexReader = IndexReader.open(store);
+      indexReader = DirectoryReader.open(store);
 
       ld = new LuceneDictionary(indexReader, "contents");
       it = ld.getWordsIterator();
 
-      // hasNext() should have no side effects
-      assertTrue("First element isn't were it should be.", it.hasNext());
-      assertTrue("First element isn't were it should be.", it.hasNext());
-      assertTrue("First element isn't were it should be.", it.hasNext());
-
       // just iterate through words
-      assertTrue("First element isn't correct", it.next().equals("Jerry"));
-      assertTrue("Second element isn't correct", it.next().equals("Tom"));
-      assertTrue("Nonexistent element is really null", it.next() == null);
-
-      // hasNext() should still have no side effects ...
-      assertFalse("There should be any more elements", it.hasNext());
-      assertFalse("There should be any more elements", it.hasNext());
-      assertFalse("There should be any more elements", it.hasNext());
-
-      // .. and there are really no more words
-      assertTrue("Nonexistent element is really null", it.next() == null);
-      assertTrue("Nonexistent element is really null", it.next() == null);
-      assertTrue("Nonexistent element is really null", it.next() == null);
+      assertEquals("First element isn't correct", "Jerry", it.next().utf8ToString());
+      assertEquals("Second element isn't correct",  "Tom", it.next().utf8ToString());
+      assertNull("Nonexistent element is really null", it.next());
     }
     finally {
       if  (indexReader != null) { indexReader.close(); }
@@ -176,15 +159,14 @@ public class TestLuceneDictionary extend
 
   public void testFieldZzz() throws IOException {
     try {
-      indexReader = IndexReader.open(store);
+      indexReader = DirectoryReader.open(store);
 
       ld = new LuceneDictionary(indexReader, "zzz");
       it = ld.getWordsIterator();
 
-      assertTrue("First element doesn't exist.", it.hasNext());
-      assertTrue("First element isn't correct", it.next().equals("bar"));
-      assertFalse("More elements than expected", it.hasNext());
-      assertTrue("Nonexistent element is really null", it.next() == null);
+      assertNotNull("First element doesn't exist.", spare = it.next());
+      assertEquals("First element isn't correct", "bar", spare.utf8ToString());
+      assertNull("More elements than expected", it.next());
     }
     finally {
       if  (indexReader != null) { indexReader.close(); }
@@ -194,7 +176,7 @@ public class TestLuceneDictionary extend
   public void testSpellchecker() throws IOException {
     Directory dir = newDirectory();
     SpellChecker sc = new SpellChecker(dir);
-    indexReader = IndexReader.open(store);
+    indexReader = DirectoryReader.open(store);
     sc.indexDictionary(new LuceneDictionary(indexReader, "contents"), newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
     String[] suggestions = sc.suggestSimilar("Tam", 1);
     assertEquals(1, suggestions.length);

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java Tue Mar  6 23:17:08 2012
@@ -412,7 +412,7 @@ public class TestSpellChecker extends Lu
     assertEquals(4, searchers.size());
     int num_field2 = this.numdoc();
     assertEquals(num_field2, num_field1 + 1);
-    int numThreads = 5 + this.random.nextInt(5);
+    int numThreads = 5 + LuceneTestCase.random.nextInt(5);
     ExecutorService executor = Executors.newFixedThreadPool(numThreads);
     SpellCheckWorker[] workers = new SpellCheckWorker[numThreads];
     for (int i = 0; i < numThreads; i++) {

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java Tue Mar  6 23:17:08 2012
@@ -97,7 +97,7 @@ public class LookupBenchmarkTest extends
     while ((line = br.readLine()) != null) {
       int tab = line.indexOf('|');
       assertTrue("No | separator?: " + line, tab >= 0);
-      float weight = Float.parseFloat(line.substring(tab + 1));
+      int weight = Integer.parseInt(line.substring(tab + 1));
       String key = line.substring(0, tab);
       input.add(new TermFreq(key, weight));
     }
@@ -191,7 +191,8 @@ public class LookupBenchmarkTest extends
 
       final List<String> input = new ArrayList<String>(benchmarkInput.size());
       for (TermFreq tf : benchmarkInput) {
-        input.add(tf.term.substring(0, Math.min(tf.term.length(), 
+        String s = tf.term.utf8ToString();
+        input.add(s.substring(0, Math.min(s.length(), 
               minPrefixLen + random.nextInt(maxPrefixLen - minPrefixLen + 1))));
       }
 
@@ -206,7 +207,7 @@ public class LookupBenchmarkTest extends
       });
 
       System.err.println(
-          String.format(Locale.ENGLISH, "%-15s queries: %d, time[ms]: %s, ~qps: %.0f",
+          String.format(Locale.ENGLISH, "%-15s queries: %d, time[ms]: %s, ~kQPS: %.0f",
               lookup.getClass().getSimpleName(),
               input.size(),
               result.average.toString(),

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java Tue Mar  6 23:17:08 2012
@@ -17,12 +17,15 @@
 package org.apache.lucene.search.suggest;
 
 import java.io.File;
+import java.util.List;
 
 import org.apache.lucene.search.suggest.Lookup;
+import org.apache.lucene.search.suggest.Lookup.LookupResult;
 import org.apache.lucene.search.suggest.fst.FSTCompletionLookup;
 import org.apache.lucene.search.suggest.jaspell.JaspellLookup;
 import org.apache.lucene.search.suggest.tst.TSTLookup;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
 
 public class PersistenceTest extends LuceneTestCase {
   public final String[] keys = new String[] {
@@ -61,7 +64,7 @@ public class PersistenceTest extends Luc
     Lookup lookup = lookupClass.newInstance();
     TermFreq[] keys = new TermFreq[this.keys.length];
     for (int i = 0; i < keys.length; i++)
-      keys[i] = new TermFreq(this.keys[i], (float) i);
+      keys[i] = new TermFreq(this.keys[i], i);
     lookup.build(new TermFreqArrayIterator(keys));
 
     // Store the suggester.
@@ -73,16 +76,18 @@ public class PersistenceTest extends Luc
     lookup.load(storeDir);
 
     // Assert validity.
-    float previous = Float.NEGATIVE_INFINITY;
+    long previous = Long.MIN_VALUE;
     for (TermFreq k : keys) {
-      Float val = (Float) lookup.get(k.term);
-      assertNotNull(k.term, val);
+      List<LookupResult> list = lookup.lookup(_TestUtil.bytesToCharSequence(k.term, random), false, 1);
+      assertEquals(1, list.size());
+      LookupResult lookupResult = list.get(0);
+      assertNotNull(k.term.utf8ToString(), lookupResult.key);
 
       if (supportsExactWeights) { 
-        assertEquals(k.term, Float.valueOf(k.v), val);
+        assertEquals(k.term.utf8ToString(), k.v, lookupResult.value);
       } else {
-        assertTrue(val + ">=" + previous, val >= previous);
-        previous = val.floatValue();
+        assertTrue(lookupResult.value + ">=" + previous, lookupResult.value >= previous);
+        previous = lookupResult.value;
       }
     }
   }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java Tue Mar  6 23:17:08 2012
@@ -1,5 +1,7 @@
 package org.apache.lucene.search.suggest;
 
+import org.apache.lucene.util.BytesRef;
+
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -18,10 +20,14 @@ package org.apache.lucene.search.suggest
  */
 
 public final class TermFreq {
-  public final String term;
-  public final float v;
+  public final BytesRef term;
+  public final long v;
 
-  public TermFreq(String term, float v) {
+  public TermFreq(String term, long v) {
+   this(new BytesRef(term), v);
+  }
+  
+  public TermFreq(BytesRef term, long v) {
     this.term = term;
     this.v = v;
   }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java Tue Mar  6 23:17:08 2012
@@ -17,10 +17,13 @@ package org.apache.lucene.search.suggest
  * limitations under the License.
  */
 
+import java.io.IOException;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.Iterator;
 
 import org.apache.lucene.search.spell.TermFreqIterator;
+import org.apache.lucene.util.BytesRef;
 
 /**
  * A {@link TermFreqIterator} over a sequence of {@link TermFreq}s.
@@ -28,6 +31,7 @@ import org.apache.lucene.search.spell.Te
 public final class TermFreqArrayIterator implements TermFreqIterator {
   private final Iterator<TermFreq> i;
   private TermFreq current;
+  private final BytesRef spare = new BytesRef();
 
   public TermFreqArrayIterator(Iterator<TermFreq> i) {
     this.i = i;
@@ -41,17 +45,22 @@ public final class TermFreqArrayIterator
     this(i.iterator());
   }
   
-  public float freq() {
+  public long weight() {
     return current.v;
   }
-  
-  public boolean hasNext() {
-    return i.hasNext();
-  }
-  
-  public String next() {
-    return (current = i.next()).term;
+
+  @Override
+  public BytesRef next() throws IOException {
+    if (i.hasNext()) {
+      current = i.next();
+      spare.copyBytes(current.term);
+      return spare;
+    }
+    return null;
   }
 
-  public void remove() { throw new UnsupportedOperationException(); }
+  @Override
+  public Comparator<BytesRef> getComparator() {
+    return null;
+  }
 }
\ No newline at end of file

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java Tue Mar  6 23:17:08 2012
@@ -17,9 +17,8 @@ package org.apache.lucene.search.suggest
  * limitations under the License.
  */
 
-import java.util.Iterator;
-
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
 import org.apache.lucene.util.LuceneTestCase;
 import org.junit.Test;
 
@@ -31,7 +30,7 @@ public class BytesRefSortersTest extends
 
   @Test
   public void testInMemorySorter() throws Exception {
-    check(new InMemorySorter());
+    check(new InMemorySorter(BytesRef.getUTF8SortedAsUnicodeComparator()));
   }
 
   private void check(BytesRefSorter sorter) throws Exception {
@@ -42,8 +41,8 @@ public class BytesRefSortersTest extends
     }
 
     // Create two iterators and check that they're aligned with each other.
-    Iterator<BytesRef> i1 = sorter.iterator();
-    Iterator<BytesRef> i2 = sorter.iterator();
+    BytesRefIterator i1 = sorter.iterator();
+    BytesRefIterator i2 = sorter.iterator();
     
     // Verify sorter contract.
     try {
@@ -52,10 +51,12 @@ public class BytesRefSortersTest extends
     } catch (IllegalStateException e) {
       // Expected.
     }
-
-    while (i1.hasNext() && i2.hasNext()) {
-      assertEquals(i1.next(), i2.next());
+    BytesRef spare1;
+    BytesRef spare2;
+    while ((spare1 = i1.next()) != null && (spare2 = i2.next()) != null) {
+      assertEquals(spare1, spare2);
     }
-    assertEquals(i1.hasNext(), i2.hasNext());
+    assertNull(i1.next());
+    assertNull(i2.next());
   }  
 }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java Tue Mar  6 23:17:08 2012
@@ -28,7 +28,7 @@ import org.apache.lucene.util.*;
  * Unit tests for {@link FSTCompletion}.
  */
 public class FSTCompletionTest extends LuceneTestCase {
-  public static TermFreq tf(String t, float v) {
+  public static TermFreq tf(String t, int v) {
     return new TermFreq(t, v);
   }
 
@@ -40,7 +40,7 @@ public class FSTCompletionTest extends L
 
     FSTCompletionBuilder builder = new FSTCompletionBuilder();
     for (TermFreq tf : evalKeys()) {
-      builder.add(new BytesRef(tf.term), (int) tf.v);
+      builder.add(tf.term, (int) tf.v);
     }
     completion = builder.build();
     completionAlphabetical = new FSTCompletion(completion.getFST(), false, true);
@@ -62,28 +62,28 @@ public class FSTCompletionTest extends L
         tf("foundation", 1),
         tf("fourblah", 1),
         tf("fourteen", 1),
-        tf("four", 0f),
-        tf("fourier", 0f),
-        tf("fourty", 0f),
+        tf("four", 0),
+        tf("fourier", 0),
+        tf("fourty", 0),
         tf("xo", 1),
       };
     return keys;
   }
 
   public void testExactMatchHighPriority() throws Exception {
-    assertMatchEquals(completion.lookup("two", 1),
+    assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("two", random), 1),
         "two/1.0");
   }
 
   public void testExactMatchLowPriority() throws Exception {
-    assertMatchEquals(completion.lookup("one", 2), 
+    assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("one", random), 2), 
         "one/0.0",
         "oneness/1.0");
   }
   
   public void testExactMatchReordering() throws Exception {
     // Check reordering of exact matches. 
-    assertMatchEquals(completion.lookup("four", 4), 
+    assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("four", random), 4), 
         "four/0.0",
         "fourblah/1.0",
         "fourteen/1.0",
@@ -92,49 +92,49 @@ public class FSTCompletionTest extends L
 
   public void testRequestedCount() throws Exception {
     // 'one' is promoted after collecting two higher ranking results.
-    assertMatchEquals(completion.lookup("one", 2), 
+    assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("one", random), 2), 
         "one/0.0", 
         "oneness/1.0");
 
     // 'four' is collected in a bucket and then again as an exact match. 
-    assertMatchEquals(completion.lookup("four", 2), 
+    assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("four", random), 2), 
         "four/0.0", 
         "fourblah/1.0");
 
     // Check reordering of exact matches. 
-    assertMatchEquals(completion.lookup("four", 4), 
+    assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("four", random), 4), 
         "four/0.0",
         "fourblah/1.0",
         "fourteen/1.0",
         "fourier/0.0");
 
     // 'one' is at the top after collecting all alphabetical results.
-    assertMatchEquals(completionAlphabetical.lookup("one", 2), 
+    assertMatchEquals(completionAlphabetical.lookup(_TestUtil.stringToCharSequence("one", random), 2), 
         "one/0.0", 
         "oneness/1.0");
     
     // 'one' is not promoted after collecting two higher ranking results.
     FSTCompletion noPromotion = new FSTCompletion(completion.getFST(), true, false);
-    assertMatchEquals(noPromotion.lookup("one", 2),  
+    assertMatchEquals(noPromotion.lookup(_TestUtil.stringToCharSequence("one", random), 2),  
         "oneness/1.0",
         "onerous/1.0");
 
     // 'one' is at the top after collecting all alphabetical results. 
-    assertMatchEquals(completionAlphabetical.lookup("one", 2), 
+    assertMatchEquals(completionAlphabetical.lookup(_TestUtil.stringToCharSequence("one", random), 2), 
         "one/0.0", 
         "oneness/1.0");
   }
 
   public void testMiss() throws Exception {
-    assertMatchEquals(completion.lookup("xyz", 1));
+    assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("xyz", random), 1));
   }
 
   public void testAlphabeticWithWeights() throws Exception {
-    assertEquals(0, completionAlphabetical.lookup("xyz", 1).size());
+    assertEquals(0, completionAlphabetical.lookup(_TestUtil.stringToCharSequence("xyz", random), 1).size());
   }
 
   public void testFullMatchList() throws Exception {
-    assertMatchEquals(completion.lookup("one", Integer.MAX_VALUE),
+    assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("one", random), Integer.MAX_VALUE),
         "oneness/1.0", 
         "onerous/1.0",
         "onesimus/1.0", 
@@ -148,7 +148,7 @@ public class FSTCompletionTest extends L
     builder.add(new BytesRef(key), 0);
 
     FSTCompletion lookup = builder.build();
-    List<Completion> result = lookup.lookup(key, 1);
+    List<Completion> result = lookup.lookup(_TestUtil.stringToCharSequence(key, random), 1);
     assertEquals(1, result.size());
   }
 
@@ -158,16 +158,16 @@ public class FSTCompletionTest extends L
     Random r = random;
     List<TermFreq> keys = new ArrayList<TermFreq>();
     for (int i = 0; i < 5000; i++) {
-      keys.add(new TermFreq(_TestUtil.randomSimpleString(r), -1.0f));
+      keys.add(new TermFreq(_TestUtil.randomSimpleString(r), -1));
     }
 
     lookup.build(new TermFreqArrayIterator(keys));
 
     // All the weights were constant, so all returned buckets must be constant, whatever they
     // are.
-    Float previous = null; 
+    Long previous = null; 
     for (TermFreq tf : keys) {
-      Float current = lookup.get(tf.term);
+      Long current = ((Number)lookup.get(_TestUtil.bytesToCharSequence(tf.term, random))).longValue();
       if (previous != null) {
         assertEquals(previous, current);
       }
@@ -175,35 +175,32 @@ public class FSTCompletionTest extends L
     }
   }  
 
-  @Nightly
   public void testMultilingualInput() throws Exception {
     List<TermFreq> input = LookupBenchmarkTest.readTop50KWiki();
 
     FSTCompletionLookup lookup = new FSTCompletionLookup();
     lookup.build(new TermFreqArrayIterator(input));
-
     for (TermFreq tf : input) {
-      assertTrue("Not found: " + tf.term, lookup.get(tf.term) != null);
-      assertEquals(tf.term, lookup.lookup(tf.term, true, 1).get(0).key);
+      assertNotNull("Not found: " + tf.term.toString(), lookup.get(_TestUtil.bytesToCharSequence(tf.term, random)));
+      assertEquals(tf.term.utf8ToString(), lookup.lookup(_TestUtil.bytesToCharSequence(tf.term, random), true, 1).get(0).key.toString());
     }
 
-    List<LookupResult> result = lookup.lookup("wit", true, 5);
+    List<LookupResult> result = lookup.lookup(_TestUtil.stringToCharSequence("wit", random), true, 5);
     assertEquals(5, result.size());
-    assertTrue(result.get(0).key.equals("wit"));  // exact match.
-    assertTrue(result.get(1).key.equals("with")); // highest count.
+    assertTrue(result.get(0).key.toString().equals("wit"));  // exact match.
+    assertTrue(result.get(1).key.toString().equals("with")); // highest count.
   }
 
   public void testEmptyInput() throws Exception {
     completion = new FSTCompletionBuilder().build();
-    assertMatchEquals(completion.lookup("", 10));
+    assertMatchEquals(completion.lookup(_TestUtil.stringToCharSequence("", random), 10));
   }
 
-  @Nightly
   public void testRandom() throws Exception {
     List<TermFreq> freqs = new ArrayList<TermFreq>();
     Random rnd = random;
     for (int i = 0; i < 2500 + rnd.nextInt(2500); i++) {
-      float weight = rnd.nextFloat() * 100; 
+      int weight = random.nextInt(100); 
       freqs.add(new TermFreq("" + rnd.nextLong(), weight));
     }
 
@@ -211,11 +208,11 @@ public class FSTCompletionTest extends L
     lookup.build(new TermFreqArrayIterator(freqs.toArray(new TermFreq[freqs.size()])));
 
     for (TermFreq tf : freqs) {
-      final String term = tf.term;
+      final String term = tf.term.utf8ToString();
       for (int i = 1; i < term.length(); i++) {
         String prefix = term.substring(0, i);
-        for (LookupResult lr : lookup.lookup(prefix, true, 10)) {
-          assertTrue(lr.key.startsWith(prefix));
+        for (LookupResult lr : lookup.lookup(_TestUtil.stringToCharSequence(prefix, random), true, 10)) {
+          assertTrue(lr.key.toString().startsWith(prefix));
         }
       }
     }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java Tue Mar  6 23:17:08 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.search.suggest
 import java.io.*;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 
 import org.apache.lucene.search.suggest.fst.Sort.BufferSize;
 import org.apache.lucene.search.suggest.fst.Sort.ByteSequencesWriter;
@@ -61,7 +62,7 @@ public class TestSort extends LuceneTest
   @Test
   public void testIntermediateMerges() throws Exception {
     // Sort 20 mb worth of data with 1mb buffer, binary merging.
-    SortInfo info = checkSort(new Sort(BufferSize.megabytes(1), Sort.defaultTempDir(), 2), 
+    SortInfo info = checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(1), Sort.defaultTempDir(), 2), 
         generateRandom(Sort.MB * 20));
     assertTrue(info.mergeRounds > 10);
   }
@@ -69,7 +70,7 @@ public class TestSort extends LuceneTest
   @Test
   public void testSmallRandom() throws Exception {
     // Sort 20 mb worth of data with 1mb buffer.
-    SortInfo sortInfo = checkSort(new Sort(BufferSize.megabytes(1), Sort.defaultTempDir(), Sort.MAX_TEMPFILES), 
+    SortInfo sortInfo = checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(1), Sort.defaultTempDir(), Sort.MAX_TEMPFILES), 
         generateRandom(Sort.MB * 20));
     assertEquals(1, sortInfo.mergeRounds);
   }
@@ -77,7 +78,7 @@ public class TestSort extends LuceneTest
   @Test @Nightly
   public void testLargerRandom() throws Exception {
     // Sort 100MB worth of data with 15mb buffer.
-    checkSort(new Sort(BufferSize.megabytes(16), Sort.defaultTempDir(), Sort.MAX_TEMPFILES), 
+    checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(16), Sort.defaultTempDir(), Sort.MAX_TEMPFILES), 
         generateRandom(Sort.MB * 100));
   }
 
@@ -92,14 +93,25 @@ public class TestSort extends LuceneTest
     byte [][] bytes = data.toArray(new byte[data.size()][]);
     return bytes;
   }
-
+  
+  static final Comparator<byte[]> unsignedByteOrderComparator = new Comparator<byte[]>() {
+    public int compare(byte[] left, byte[] right) {
+      final int max = Math.min(left.length, right.length);
+      for (int i = 0, j = 0; i < max; i++, j++) {
+        int diff = (left[i]  & 0xff) - (right[j] & 0xff); 
+        if (diff != 0) 
+          return diff;
+      }
+      return left.length - right.length;
+    }
+  };
   /**
    * Check sorting data on an instance of {@link Sort}.
    */
   private SortInfo checkSort(Sort sort, byte[][] data) throws IOException {
     File unsorted = writeAll("unsorted", data);
 
-    Arrays.sort(data, Sort.unsignedByteOrderComparator);
+    Arrays.sort(data, unsignedByteOrderComparator);
     File golden = writeAll("golden", data);
 
     File sorted = new File(tempDir, "sorted");

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java Tue Mar  6 23:17:08 2012
@@ -45,33 +45,33 @@ public class WFSTCompletionTest extends 
     suggester.build(new TermFreqArrayIterator(keys));
     
     // top N of 2, but only foo is available
-    List<LookupResult> results = suggester.lookup("f", false, 2);
+    List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("f", random), false, 2);
     assertEquals(1, results.size());
-    assertEquals("foo", results.get(0).key);
+    assertEquals("foo", results.get(0).key.toString());
     assertEquals(50, results.get(0).value, 0.01F);
     
     // top N of 1 for 'bar': we return this even though barbar is higher
-    results = suggester.lookup("bar", false, 1);
+    results = suggester.lookup(_TestUtil.stringToCharSequence("bar", random), false, 1);
     assertEquals(1, results.size());
-    assertEquals("bar", results.get(0).key);
+    assertEquals("bar", results.get(0).key.toString());
     assertEquals(10, results.get(0).value, 0.01F);
     
     // top N Of 2 for 'b'
-    results = suggester.lookup("b", false, 2);
+    results = suggester.lookup(_TestUtil.stringToCharSequence("b", random), false, 2);
     assertEquals(2, results.size());
-    assertEquals("barbar", results.get(0).key);
+    assertEquals("barbar", results.get(0).key.toString());
     assertEquals(12, results.get(0).value, 0.01F);
-    assertEquals("bar", results.get(1).key);
+    assertEquals("bar", results.get(1).key.toString());
     assertEquals(10, results.get(1).value, 0.01F);
     
     // top N of 3 for 'ba'
-    results = suggester.lookup("ba", false, 3);
+    results = suggester.lookup(_TestUtil.stringToCharSequence("ba", random), false, 3);
     assertEquals(3, results.size());
-    assertEquals("barbar", results.get(0).key);
+    assertEquals("barbar", results.get(0).key.toString());
     assertEquals(12, results.get(0).value, 0.01F);
-    assertEquals("bar", results.get(1).key);
+    assertEquals("bar", results.get(1).key.toString());
     assertEquals(10, results.get(1).value, 0.01F);
-    assertEquals("barbara", results.get(2).key);
+    assertEquals("barbara", results.get(2).key.toString());
     assertEquals(6, results.get(2).value, 0.01F);
   }
   
@@ -100,7 +100,7 @@ public class WFSTCompletionTest extends 
       // we can probably do Integer.MAX_VALUE here, but why worry.
       int weight = random.nextInt(1<<24);
       slowCompletor.put(s, (long)weight);
-      keys[i] = new TermFreq(s, (float) weight);
+      keys[i] = new TermFreq(s, weight);
     }
 
     WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
@@ -109,7 +109,7 @@ public class WFSTCompletionTest extends 
     for (String prefix : allPrefixes) {
     
       final int topN = _TestUtil.nextInt(random, 1, 10);
-      List<LookupResult> r = suggester.lookup(prefix, false, topN);
+      List<LookupResult> r = suggester.lookup(_TestUtil.stringToCharSequence(prefix, random), false, topN);
 
       // 2. go thru whole treemap (slowCompletor) and check its actually the best suggestion
       final List<LookupResult> matches = new ArrayList<LookupResult>();
@@ -117,7 +117,7 @@ public class WFSTCompletionTest extends 
       // TODO: could be faster... but its slowCompletor for a reason
       for (Map.Entry<String,Long> e : slowCompletor.entrySet()) {
         if (e.getKey().startsWith(prefix)) {
-          matches.add(new LookupResult(e.getKey(), (float)e.getValue().longValue()));
+          matches.add(new LookupResult(e.getKey(), e.getValue().longValue()));
         }
       }
 
@@ -126,7 +126,7 @@ public class WFSTCompletionTest extends 
         public int compare(LookupResult left, LookupResult right) {
           int cmp = Float.compare(right.value, left.value);
           if (cmp == 0) {
-            return left.key.compareTo(right.key);
+            return left.compareTo(right);
           } else {
             return cmp;
           }
@@ -140,7 +140,7 @@ public class WFSTCompletionTest extends 
 
       for(int hit=0;hit<r.size();hit++) {
         //System.out.println("  check hit " + hit);
-        assertEquals(matches.get(hit).key, r.get(hit).key);
+        assertEquals(matches.get(hit).key.toString(), r.get(hit).key.toString());
         assertEquals(matches.get(hit).value, r.get(hit).value, 0f);
       }
     }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/CHANGES.txt?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/CHANGES.txt Tue Mar  6 23:17:08 2012
@@ -1,4 +1,4 @@
-                      Apache Solr Release Notes
+                      Apache Solr Release Notes
 
 Introduction
 ------------
@@ -104,6 +104,8 @@ New Features
   (Stefan Matheis, Mark Miller)
   SOLR-3108: Error in SolrCloud's replica lookup code when replica's are hosted in same Solr instance.
   (Bruno Dumon, Sami Siren, Mark Miller)
+  SOLR-3080: Remove shard info from zookeeper when SolrCore is explicitlyunloaded.
+  (yonik, Mark Miller, siren)
 
 * SOLR-1566: Transforming documents in the ResponseWriters.  This will allow
   for more complex results in responses and open the door for function queries
@@ -224,6 +226,12 @@ New Features
 * SOLR-3120: Optional post filtering for spatial queries bbox and geofilt
   for LatLonType. (yonik)
 
+* SOLR-2459: Expose LogLevel selection with a RequestHandler rather then servlet
+  (Stefan Matheis, Upayavira, ryan)
+
+* SOLR-3134: Include shard info in distributed response when shards.info=true 
+  (Russell Black, ryan)
+
 
 Optimizations
 ----------------------
@@ -256,6 +264,9 @@ Optimizations
 Bug Fixes
 ----------------------
 
+* SOLR-3165: Cannot use DIH in Solrcloud + Zookeeper (Alexey Serba, 
+  Mark Miller, siren)
+
 * SOLR-3068: Occasional NPE in ThreadDumpHandler (siren)
 
 * SOLR-2762: FSTLookup could return duplicate results or one results less
@@ -318,7 +329,7 @@ Bug Fixes
 * SOLR-1520: QueryElevationComponent now supports non-string ids (gsingers)
 
 * SOLR-3037: When using binary format in solrj the codec screws up parameters
-  (Sami Siren via yonik)
+  (Sami Siren, Jörg Maier  via yonik)
 
 * SOLR-3062: A join in the main query was not respecting any filters pushed
   down to it via acceptDocs since LUCENE-1536. (Mike Hugo, yonik)
@@ -425,6 +436,10 @@ Other Changes
 * SOLR-3005: Default QueryResponseWriters are now initialized via init() with an empty
   NamedList. (Gasol Wu, Chris Male)
 
+* SOLR-2607: Removed obsolete client/ folder (ehatcher, Eric Pugh, janhoy)
+
+* SOLR-3202: Dropping Support for JSP.  New Admin UI is all client side (ryan)
+
 
 Documentation
 ----------------------
@@ -447,6 +462,9 @@ Upgrading from Solr 3.5
   HTMLStripCharFilter in Solr version 3.5 and earlier: the old implementation
   (bugs and all) is preserved as LegacyHTMLStripCharFilter.
 
+* SOLR-3040: The DIH's admin UI (dataimport.jsp) now requires DIH request handlers to start with
+  a '/'. (dsmiley)
+
 New Features
 ----------------------
 * SOLR-2904: BinaryUpdateRequestHandler should be able to accept multiple update requests from
@@ -509,6 +527,9 @@ New Features
 * SOLR-3143: Add SuggestQueryConverter, a QueryConverter intended for
   auto-suggesters. (Robert Muir)
 
+* SOLR-3033: ReplicationHandler's backup command now supports a 'maxNumberOfBackups' 
+  init param that can be used to delete all but the most recent N backups. (Torsten Krah, James Dyer)
+
 Optimizations
 ----------------------
 * SOLR-1931: Speedup for LukeRequestHandler and admin/schema browser. New parameter
@@ -525,6 +546,8 @@ Optimizations
 
 Bug Fixes
 ----------------------
+* SOLR-3187 SystemInfoHandler leaks filehandles (siren)
+
 * SOLR-2912: Fixed File descriptor leak in ShowFileRequestHandler (Michael Ryan, shalin)
 
 * SOLR-2819: Improved speed of parsing hex entities in HTMLStripCharFilter
@@ -608,6 +631,15 @@ Bug Fixes
 * SOLR-3052: Fixed typo in distributed grouping parameters.
   (Martijn van Groningen, Grant Ingersoll)
 
+* SOLR-2909: Add support for ResourceLoaderAware tokenizerFactories in synonym
+  filter factories. (Tom Klonikowski, Jun Ohtani via Koji Sekiguchi)
+  
+* SOLR-3168: ReplicationHandler "numberToKeep" & "maxNumberOfBackups" parameters
+  would keep only 1 backup, even if more than 1 was specified (Neil Hooey, James Dyer)
+
+* SOLR-3195: timeAllowed is ignored for grouping queries
+  (Russell Black via Martijn van Groningen)
+
 Other Changes
 ----------------------
 * SOLR-2922: Upgrade commons-io and commons-lang to 2.1 and 2.6, respectively. (koji)
@@ -667,7 +699,7 @@ New Features
   (Chris Male, Mark Holland, Gunnlaugur Thor Briem, Ryan McKinley)
 
 * SOLR-2578: ReplicationHandler's backup command now supports a 'numberToKeep' 
-  param that can be used to delete all but the most recent N backups.
+  request param that can be used to delete all but the most recent N backups.
   (James Dyer via hossman)
 
 Optimizations
@@ -1789,6 +1821,10 @@ Other Changes
   servlet containers was therefore removed and is now ignored if set.
   Output is always UTF-8.  (uschindler, yonik, rmuir)
 
+* SOLR-141: Errors and Exceptions are formated by ResponseWriter.
+  (Mike Sokolov, Rich Cariens, Daniel Naber, ryan)
+
+
 Build
 ----------------------
 

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/README.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/README.txt?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/README.txt (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/README.txt Tue Mar  6 23:17:08 2012
@@ -32,7 +32,8 @@ See the "example" directory for an examp
 using the example setup can be found at
 http://lucene.apache.org/solr/tutorial.html
 or in in "docs/tutorial.html" in a binary distribution.
-
+Also, there are Solr clients for many programming languages, see 
+http://wiki.apache.org/solr/IntegratingSolr
 
 
 Files included in an Apache Solr binary distribution

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/build.xml?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/build.xml (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/build.xml Tue Mar  6 23:17:08 2012
@@ -130,7 +130,7 @@
   <target name="compile" description="Compile the source code."
           depends="compile-core, compile-contrib"/>
   <target name="test" description="Validate, then run core, solrj, and contrib unit tests."
-          depends="validate, test-jsp, test-core, test-contrib"/>
+          depends="validate, test-core, test-contrib"/>
   <target name="test-core" description="Runs the core and solrj unit tests."
           depends="test-solr-core, test-solrj"/>
   <target name="compile-test" description="Compile unit tests."
@@ -169,14 +169,7 @@
       <propertyset refid="uptodate.and.compiled.properties"/>
     </ant>
   </target>
-  
-  <!-- Webapp targets -->
-  <target name="test-jsp">
-    <ant dir="webapp" target="test" inheritall="false">
-      <propertyset refid="uptodate.and.compiled.properties"/>
-    </ant>
-  </target>
-  
+    
   <!-- Validation (license/ notice checks). -->
   <target name="validate" depends="compile-tools" description="Validate legal stuff.">
     <license-check-macro dir="${basedir}">
@@ -189,8 +182,6 @@
       <additional-filters>
         <replaceregex pattern="/jetty-util([^/]+)$" replace="/jetty-util" flags="gi" />
         <replaceregex pattern="/jetty-6([^/]+)$" replace="/jetty" flags="gi" />
-        <replaceregex pattern="/jsp-2.1-glassfish([^/]+)$" replace="/jsp-2.1-glassfish" flags="gi" />
-        <replaceregex pattern="/jsp-api-2.1-glassfish([^/]+)$" replace="/jsp-api-2.1-glassfish" flags="gi" />
       </additional-filters>
     </license-check-macro>
   </target>

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/common-build.xml?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/common-build.xml (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/common-build.xml Tue Mar  6 23:17:08 2012
@@ -49,8 +49,8 @@
   <property name="tests.loggingfile" value="${common-solr.dir}/testlogging.properties"/>
   <property name="tests.cleanthreads.sysprop" value="perClass"/>
 
-  <property name="clover.db.dir" location="${dest}/tests/clover/db"/>
-  <property name="clover.report.dir" location="${dest}/tests/clover/reports"/>
+  <property name="clover.db.dir" location="${dest}/test/clover/db"/>
+  <property name="clover.report.dir" location="${dest}/test/clover/reports"/>
   <available property="clover.present" classname="com.cenqua.clover.tasks.CloverReportTask"/>
   <condition property="clover.enabled">
     <and>
@@ -83,6 +83,7 @@
   	<pathelement path="${analyzers-common.jar}"/>
   	<pathelement path="${analyzers-kuromoji.jar}"/>
   	<pathelement path="${analyzers-phonetic.jar}"/>
+    <pathelement path="${analyzers-uima.jar}"/>
   	<pathelement path="${highlighter.jar}"/>
   	<pathelement path="${memory.jar}"/>
   	<pathelement path="${misc.jar}"/>

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java Tue Mar  6 23:17:08 2012
@@ -19,7 +19,9 @@ package org.apache.solr.handler.dataimpo
 import com.sun.mail.imap.IMAPMessage;
 
 import org.apache.tika.Tika;
+import org.apache.tika.metadata.HttpHeaders;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaMetadataKeys;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -169,8 +171,8 @@ public class MailEntityProcessor extends
       InputStream is = part.getInputStream();
       String fileName = part.getFileName();
       Metadata md = new Metadata();
-      md.set(Metadata.CONTENT_TYPE, ctype.getBaseType().toLowerCase(Locale.ENGLISH));
-      md.set(Metadata.RESOURCE_NAME_KEY, fileName);
+      md.set(HttpHeaders.CONTENT_TYPE, ctype.getBaseType().toLowerCase(Locale.ENGLISH));
+      md.set(TikaMetadataKeys.RESOURCE_NAME_KEY, fileName);
       String content = tika.parseToString(is, md);
       if (disp != null && disp.equalsIgnoreCase(Part.ATTACHMENT)) {
         if (row.get(ATTACHMENT) == null)

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/TikaEntityProcessor.java Tue Mar  6 23:17:08 2012
@@ -34,6 +34,7 @@ import org.xml.sax.helpers.DefaultHandle
 
 import javax.xml.transform.OutputKeys;
 import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerFactory;
 import javax.xml.transform.sax.SAXTransformerFactory;
 import javax.xml.transform.sax.TransformerHandler;
 import javax.xml.transform.stream.StreamResult;
@@ -142,7 +143,7 @@ public class TikaEntityProcessor extends
   private static ContentHandler getHtmlHandler(Writer writer)
           throws TransformerConfigurationException {
     SAXTransformerFactory factory = (SAXTransformerFactory)
-            SAXTransformerFactory.newInstance();
+            TransformerFactory.newInstance();
     TransformerHandler handler = factory.newTransformerHandler();
     handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "html");
     handler.setResult(new StreamResult(writer));
@@ -185,7 +186,7 @@ public class TikaEntityProcessor extends
   private static ContentHandler getXmlContentHandler(Writer writer)
           throws TransformerConfigurationException {
     SAXTransformerFactory factory = (SAXTransformerFactory)
-            SAXTransformerFactory.newInstance();
+            TransformerFactory.newInstance();
     TransformerHandler handler = factory.newTransformerHandler();
     handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
     handler.setResult(new StreamResult(writer));

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/CHANGES.txt?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/CHANGES.txt Tue Mar  6 23:17:08 2012
@@ -18,6 +18,10 @@ New Features
 ----------------------
 * SOLR-1499: Added SolrEntityProcessor that imports data from another Solr core or instance based on a specified query.
              (Lance Norskog, Erik Hatcher, Pulkit Singhal, Ahmet Arslan, Luca Cavanna, Martijn van Groningen)
+             Additional Work:
+             SOLR-3190: Minor improvements to SolrEntityProcessor. Add more consistency between solr parameters
+             and parameters used in SolrEntityProcessor and ability to specify a custom HttpClient instance.
+             (Luca Cavanna via Martijn van Groningen)
 
 Changes in Runtime Behavior
 ----------------------

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java Tue Mar  6 23:17:08 2012
@@ -28,7 +28,6 @@ import org.apache.solr.common.util.Named
 import org.apache.solr.common.util.ContentStream;
 import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.common.util.SystemIdResolver;
-import org.apache.solr.core.SolrConfig;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.core.SolrResourceLoader;
 import org.apache.solr.handler.RequestHandlerBase;
@@ -39,7 +38,6 @@ import org.apache.solr.response.SolrQuer
 import org.apache.solr.request.SolrRequestHandler;
 import org.apache.solr.update.processor.UpdateRequestProcessor;
 import org.apache.solr.update.processor.UpdateRequestProcessorChain;
-import org.apache.solr.util.SolrPluginUtils;
 import org.apache.solr.util.plugin.SolrCoreAware;
 
 import java.util.*;
@@ -109,7 +107,7 @@ public class DataImportHandler extends R
         String configLoc = (String) defaults.get("config");
         if (configLoc != null && configLoc.length() != 0) {
           processConfiguration(defaults);
-          final InputSource is = new InputSource(core.getResourceLoader().openConfig(configLoc));
+          final InputSource is = new InputSource(core.getResourceLoader().openResource(configLoc));
           is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(configLoc));
           importer = new DataImporter(is, core,
                   dataSources, coreScopeSession, myName);

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java Tue Mar  6 23:17:08 2012
@@ -19,7 +19,6 @@ package org.apache.solr.handler.dataimpo
 
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.core.SolrConfig;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.schema.SchemaField;
@@ -96,11 +95,20 @@ public class DataImporter {
    */
   DataImporter() {
     coreScopeSession = new ConcurrentHashMap<String, Object>();
-    this.propWriter = new SimplePropertiesWriter();
+    createPropertyWriter();
     propWriter.init(this);
     this.handlerName = "dataimport" ;
   }
 
+  private void createPropertyWriter() {
+    if (this.core == null
+        || !this.core.getCoreDescriptor().getCoreContainer().isZooKeeperAware()) {
+      propWriter = new SimplePropertiesWriter();
+    } else {
+      propWriter = new ZKPropertiesWriter();
+    }
+  }
+
   DataImporter(InputSource dataConfig, SolrCore core, Map<String, Properties> ds, Map<String, Object> session, String handlerName) {
       this.handlerName = handlerName;
     if (dataConfig == null)
@@ -108,7 +116,7 @@ public class DataImporter {
               "Configuration not found");
     this.core = core;
     this.schema = core.getSchema();
-    this.propWriter = new SimplePropertiesWriter();
+    createPropertyWriter();
     propWriter.init(this);
     dataSourceProps = ds;
     if (session == null)



Mime
View raw message