incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [14/51] [partial] Initial repackage to org.apache.
Date Mon, 03 Sep 2012 03:17:11 GMT
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/org/apache/blur/utils/QueryCache.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/QueryCache.java b/src/blur-core/src/main/java/org/apache/blur/utils/QueryCache.java
new file mode 100644
index 0000000..7d5eff2
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/QueryCache.java
@@ -0,0 +1,107 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.SortedSet;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurResults;
+
+import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
+import com.googlecode.concurrentlinkedhashmap.EvictionListener;
+
+public class QueryCache implements EvictionListener<QueryCacheKey, QueryCacheEntry> {
+
+  private static final Log LOG = LogFactory.getLog(QueryCache.class);
+
+  private long _ttl;
+  private String _name;
+  private int _cachedElements;
+  private ConcurrentLinkedHashMap<QueryCacheKey, QueryCacheEntry> _cache;
+
+  public QueryCache(String name, int cachedElements, long ttl) {
+    _name = name;
+    _cachedElements = cachedElements;
+    _ttl = ttl;
+    _cache = new ConcurrentLinkedHashMap.Builder<QueryCacheKey, QueryCacheEntry>().maximumWeightedCapacity(_cachedElements).listener(this).build();
+  }
+
+  @Override
+  public void onEviction(QueryCacheKey key, QueryCacheEntry value) {
+    LOG.debug("Cache [" + _name + "] key [" + key + "] value [" + value + "] evicted.");
+  }
+
+  @SuppressWarnings("deprecation")
+  public static QueryCacheKey getNormalizedBlurQueryKey(String table, BlurQuery blurQuery) {
+    BlurQuery newBlurQuery = new BlurQuery(blurQuery);
+    newBlurQuery.allowStaleData = false;
+    newBlurQuery.useCacheIfPresent = false;
+    newBlurQuery.userContext = null;
+    newBlurQuery.maxQueryTime = 0;
+    newBlurQuery.uuid = 0;
+    newBlurQuery.startTime = 0;
+    newBlurQuery.modifyFileCaches = false;
+    return new QueryCacheKey(table, newBlurQuery);
+  }
+
+  public boolean isValid(QueryCacheEntry entry, SortedSet<String> currentShards) {
+    if (!isValid(entry)) {
+      return false;
+    }
+    if (!entry.shards.equals(currentShards)) {
+      return false;
+    }
+    return true;
+  }
+
+  public boolean isValid(QueryCacheEntry entry) {
+    if (entry == null) {
+      return false;
+    }
+    if (entry.timestamp + _ttl < System.currentTimeMillis()) {
+      return false;
+    }
+    return true;
+  }
+
+  public BlurResults cache(String table, BlurQuery original, BlurResults results) {
+    if (results == null) {
+      return null;
+    }
+    if (original != null && original.cacheResult) {
+      LOG.debug("Caching results for query [{0}]", original);
+      BlurResults cacheResults = new BlurResults(results);
+      cacheResults.query = null;
+      put(getNormalizedBlurQueryKey(table, original), new QueryCacheEntry(cacheResults));
+    }
+    return results;
+  }
+
+  public void put(QueryCacheKey key, QueryCacheEntry value) {
+    _cache.put(key, value);
+  }
+
+  public QueryCacheEntry get(QueryCacheKey key) {
+    return _cache.get(key);
+  }
+
+  public void remove(QueryCacheKey key) {
+    _cache.remove(key);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/org/apache/blur/utils/QueryCacheEntry.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/QueryCacheEntry.java b/src/blur-core/src/main/java/org/apache/blur/utils/QueryCacheEntry.java
new file mode 100644
index 0000000..8b66f46
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/QueryCacheEntry.java
@@ -0,0 +1,46 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurResults;
+
+
+public class QueryCacheEntry {
+  public BlurResults results;
+  public long timestamp;
+  public SortedSet<String> shards;
+
+  public QueryCacheEntry(BlurResults cacheResults) {
+    results = cacheResults;
+    timestamp = System.currentTimeMillis();
+    if (cacheResults != null && cacheResults.shardInfo != null) {
+      shards = new TreeSet<String>(cacheResults.shardInfo.keySet());
+    } else {
+      shards = new TreeSet<String>();
+    }
+  }
+
+  public BlurResults getBlurResults(BlurQuery blurQuery) {
+    BlurResults blurResults = new BlurResults(results);
+    blurResults.query = blurQuery;
+    return blurResults;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/org/apache/blur/utils/QueryCacheKey.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/QueryCacheKey.java b/src/blur-core/src/main/java/org/apache/blur/utils/QueryCacheKey.java
new file mode 100644
index 0000000..e17aa23
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/QueryCacheKey.java
@@ -0,0 +1,77 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.blur.thrift.generated.BlurQuery;
+
+public class QueryCacheKey {
+  private String table;
+  private BlurQuery query;
+
+  public String getTable() {
+    return table;
+  }
+
+  public void setTable(String table) {
+    this.table = table;
+  }
+
+  public BlurQuery getQuery() {
+    return query;
+  }
+
+  public void setQuery(BlurQuery query) {
+    this.query = query;
+  }
+
+  public QueryCacheKey(String table, BlurQuery query) {
+    this.table = table;
+    this.query = query;
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + ((query == null) ? 0 : query.hashCode());
+    result = prime * result + ((table == null) ? 0 : table.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    QueryCacheKey other = (QueryCacheKey) obj;
+    if (query == null) {
+      if (other.query != null)
+        return false;
+    } else if (!query.equals(other.query))
+      return false;
+    if (table == null) {
+      if (other.table != null)
+        return false;
+    } else if (!table.equals(other.table))
+      return false;
+    return true;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/org/apache/blur/utils/ReaderBlurRecord.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/ReaderBlurRecord.java b/src/blur-core/src/main/java/org/apache/blur/utils/ReaderBlurRecord.java
new file mode 100644
index 0000000..fb31401
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/ReaderBlurRecord.java
@@ -0,0 +1,29 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+public interface ReaderBlurRecord {
+
+  void setRecordIdStr(String value);
+
+  void addColumn(String name, String value);
+
+  void setFamilyStr(String family);
+
+  void setRowIdStr(String rowId);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/org/apache/blur/utils/ReferenceIterable.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/ReferenceIterable.java b/src/blur-core/src/main/java/org/apache/blur/utils/ReferenceIterable.java
new file mode 100644
index 0000000..e537e2f
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/ReferenceIterable.java
@@ -0,0 +1,55 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+public class ReferenceIterable<T> implements Iterable<T> {
+
+  private T t;
+
+  public ReferenceIterable(T t) {
+    this.t = t;
+  }
+
+  @Override
+  public Iterator<T> iterator() {
+    return new Iterator<T>() {
+      boolean taken = false;
+
+      @Override
+      public boolean hasNext() {
+        return !taken;
+      }
+
+      @Override
+      public T next() {
+        if (taken) {
+          throw new NoSuchElementException();
+        }
+        taken = true;
+        return t;
+      }
+
+      @Override
+      public void remove() {
+        throw new UnsupportedOperationException();
+      }
+    };
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java b/src/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java
new file mode 100644
index 0000000..25bca7d
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java
@@ -0,0 +1,94 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.apache.blur.utils.BlurConstants.RECORD_ID;
+import static org.apache.blur.utils.BlurConstants.ROW_ID;
+import static org.apache.blur.utils.BlurConstants.SEP;
+
+import java.util.ArrayList;
+
+import org.apache.blur.thrift.generated.FetchRecordResult;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.Row;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Fieldable;
+
+
+public class RowDocumentUtil {
+
+  public static FetchRecordResult getColumns(Document document) {
+    FetchRecordResult result = new FetchRecordResult();
+    BlurThriftRecord record = new BlurThriftRecord();
+    String rowId = readRecord(document, record);
+    result.setRecord(record);
+    result.setRowid(rowId);
+    return result;
+  }
+
+  public static Row getRow(Iterable<Document> docs) {
+    Row row = new Row();
+    boolean empty = true;
+    if (docs == null) {
+      return null;
+    }
+    for (Document document : docs) {
+      empty = false;
+      BlurThriftRecord record = new BlurThriftRecord();
+      String rowId = readRecord(document, record);
+      if (record.getColumns() != null) {
+        row.addToRecords(record);
+      }
+      if (row.id == null) {
+        row.setId(rowId);
+      }
+      row.recordCount++;
+    }
+    if (empty) {
+      return null;
+    }
+    if (row.records == null) {
+      row.records = new ArrayList<Record>();
+    }
+    return row;
+  }
+
+  public static String readRecord(Document document, ReaderBlurRecord reader) {
+    String rowId = null;
+    String family = null;
+    for (Fieldable field : document.getFields()) {
+      if (field.name().equals(ROW_ID)) {
+        rowId = field.stringValue();
+      } else if (field.name().equals(RECORD_ID)) {
+        reader.setRecordIdStr(field.stringValue());
+      } else {
+        String name = field.name();
+        int index = name.indexOf(SEP);
+        if (index < 0) {
+          continue;
+        } else if (family == null) {
+          family = name.substring(0, index);
+        }
+        name = name.substring(index + 1);
+        reader.addColumn(name, field.stringValue());
+      }
+    }
+    reader.setFamilyStr(family);
+    reader.setRowIdStr(rowId);
+    return rowId;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/org/apache/blur/utils/RowIndexWriter.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/RowIndexWriter.java b/src/blur-core/src/main/java/org/apache/blur/utils/RowIndexWriter.java
new file mode 100644
index 0000000..0f187c5
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/RowIndexWriter.java
@@ -0,0 +1,143 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.apache.blur.utils.BlurConstants.RECORD_ID;
+import static org.apache.blur.utils.BlurConstants.ROW_ID;
+import static org.apache.blur.utils.BlurConstants.SEP;
+import static org.apache.blur.utils.BlurConstants.SUPER;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.blur.analysis.FieldConverterUtil;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.Row;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+
+
+public class RowIndexWriter {
+
+  private BlurAnalyzer _analyzer;
+  private IndexWriter _indexWriter;
+  private boolean primeDocSet;
+  private StringBuilder builder = new StringBuilder();
+
+  public RowIndexWriter(IndexWriter indexWriter, BlurAnalyzer analyzer) {
+    _indexWriter = indexWriter;
+    _analyzer = analyzer;
+  }
+
+  public void add(boolean wal, Row row) throws IOException {
+    if (row == null || row.id == null) {
+      throw new NullPointerException();
+    }
+    append(wal, row, false);
+  }
+
+  public void replace(boolean wal, Row row) throws IOException {
+    if (row == null || row.id == null) {
+      throw new NullPointerException();
+    }
+    append(wal, row, true);
+  }
+
+  private void append(boolean wal, Row row, boolean replace) throws IOException {
+    primeDocSet = false;
+    List<Document> documents = new ArrayList<Document>();
+    for (Record record : row.records) {
+      convert(row.id, record, documents);
+    }
+    if (replace) {
+      _indexWriter.updateDocuments(new Term(ROW_ID, row.id), documents, _analyzer);
+    } else {
+      _indexWriter.addDocuments(documents, _analyzer);
+    }
+  }
+
+  private void convert(String rowId, Record record, List<Document> documents) throws IOException {
+    if (record == null) {
+      return;
+    }
+    String recordId = record.recordId;
+    if (recordId == null) {
+      throw new NullPointerException("Record id is null.");
+    }
+    String family = record.getFamily();
+    if (family == null) {
+      throw new NullPointerException("Family is null.");
+    }
+    Document document = new Document();
+    document.add(new Field(ROW_ID, rowId, Store.YES, Index.NOT_ANALYZED_NO_NORMS));
+    document.add(new Field(RECORD_ID, recordId, Store.YES, Index.NOT_ANALYZED_NO_NORMS));
+    if (addColumns(document, _analyzer, builder, family, record.columns)) {
+      FieldConverterUtil.convert(document, _analyzer);
+      if (!primeDocSet) {
+        document.add(BlurConstants.PRIME_DOC_FIELD);
+        primeDocSet = true;
+      }
+      documents.add(document);
+    }
+  }
+
+  public static boolean addColumns(Document document, BlurAnalyzer analyzer, StringBuilder builder, String columnFamily, Iterable<Column> set) {
+    if (set == null) {
+      return false;
+    }
+    builder.setLength(0);
+    OUTER: for (Column column : set) {
+      String name = column.getName();
+      String value = column.value;
+      if (value == null || name == null) {
+        continue OUTER;
+      }
+      String fieldName = getFieldName(columnFamily, name);
+      Store store = analyzer.getStore(fieldName);
+      Index index = analyzer.getIndex(fieldName);
+      boolean fullText = analyzer.isFullTextField(fieldName);
+      Set<String> subFieldNames = analyzer.getSubIndexNames(fieldName);
+      document.add(new Field(fieldName, value, store, index));
+      if (fullText) {
+        builder.append(value).append(' ');
+      }
+      if (subFieldNames != null) {
+        for (String subFieldName : subFieldNames) {
+          document.add(new Field(subFieldName, value, Store.NO, index));
+        }
+      }
+    }
+    if (builder.length() != 0) {
+      String superValue = builder.toString();
+      document.add(new Field(SUPER, superValue, Store.NO, Index.ANALYZED_NO_NORMS));
+    }
+    return true;
+  }
+
+  public static String getFieldName(String columnFamily, String name) {
+    return columnFamily + SEP + name;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java b/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
new file mode 100644
index 0000000..8312360
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
@@ -0,0 +1,91 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.TermDocs;
+
+public class TermDocIterable implements Iterable<Document> {
+
+  private TermDocs termDocs;
+  private IndexReader reader;
+  private FieldSelector fieldSelector;
+
+  public TermDocIterable(TermDocs termDocs, IndexReader reader, FieldSelector fieldSelector) {
+    this.termDocs = termDocs;
+    this.reader = reader;
+    this.fieldSelector = fieldSelector;
+  }
+
+  @Override
+  public Iterator<Document> iterator() {
+    return new Iterator<Document>() {
+      private boolean hasNext = getNext();
+
+      @Override
+      public boolean hasNext() {
+        return hasNext;
+      }
+
+      @Override
+      public Document next() {
+        Document doc;
+        try {
+          doc = getDoc();
+          hasNext = getNext();
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+        return doc;
+      }
+
+      @Override
+      public void remove() {
+
+      }
+    };
+  }
+
+  private Document getDoc() throws IOException {
+    return reader.document(termDocs.doc(), fieldSelector);
+  }
+
+  private boolean getNext() {
+    try {
+      boolean next = termDocs.next();
+      if (!next) {
+        termDocs.close();
+        return next;
+      }
+      while (reader.isDeleted(termDocs.doc())) {
+        next = termDocs.next();
+      }
+      if (!next) {
+        termDocs.close();
+      }
+      return next;
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/test/java/com/nearinfinity/blur/MiniCluster.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/com/nearinfinity/blur/MiniCluster.java b/src/blur-core/src/test/java/com/nearinfinity/blur/MiniCluster.java
deleted file mode 100644
index d61e2d9..0000000
--- a/src/blur-core/src/test/java/com/nearinfinity/blur/MiniCluster.java
+++ /dev/null
@@ -1,465 +0,0 @@
-package com.nearinfinity.blur;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static com.nearinfinity.blur.utils.BlurConstants.*;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COUNT;
-import static com.nearinfinity.blur.utils.BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ThreadPoolExecutor;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.thrift.TException;
-import org.apache.thrift.transport.TTransportException;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.server.ServerConfig;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;
-
-import com.nearinfinity.blur.log.Log;
-import com.nearinfinity.blur.log.LogFactory;
-import com.nearinfinity.blur.thrift.BlurClient;
-import com.nearinfinity.blur.thrift.BlurClientManager;
-import com.nearinfinity.blur.thrift.Connection;
-import com.nearinfinity.blur.thrift.ThriftBlurControllerServer;
-import com.nearinfinity.blur.thrift.ThriftBlurShardServer;
-import com.nearinfinity.blur.thrift.ThriftServer;
-import com.nearinfinity.blur.thrift.generated.AnalyzerDefinition;
-import com.nearinfinity.blur.thrift.generated.Blur.Client;
-import com.nearinfinity.blur.thrift.generated.Blur.Iface;
-import com.nearinfinity.blur.thrift.generated.BlurException;
-import com.nearinfinity.blur.thrift.generated.BlurQuery;
-import com.nearinfinity.blur.thrift.generated.BlurResults;
-import com.nearinfinity.blur.thrift.generated.Column;
-import com.nearinfinity.blur.thrift.generated.Record;
-import com.nearinfinity.blur.thrift.generated.Row;
-import com.nearinfinity.blur.thrift.generated.RowMutation;
-import com.nearinfinity.blur.thrift.generated.TableDescriptor;
-import com.nearinfinity.blur.utils.BlurUtil;
-
-public abstract class MiniCluster {
-
-  private static Log LOG = LogFactory.getLog(MiniCluster.class);
-  private static MiniDFSCluster cluster;
-  private static Thread serverThread;
-  private static String zkConnectionString = "localhost:21810";
-  private static ZooKeeperServerMainEmbedded zooKeeperServerMain;
-  private static List<ThriftServer> controllers = new ArrayList<ThriftServer>();
-  private static List<ThriftServer> shards = new ArrayList<ThriftServer>();
-  private static String controllerConnectionStr;
-
-  public static void main(String[] args) throws IOException, InterruptedException, KeeperException, BlurException, TException {
-    startDfs("./tmp");
-    startZooKeeper("./tmp");
-    startControllers(1);
-    startShards(1);
-
-    try {
-      Iface client = BlurClient.getClient(getControllerConnectionStr());
-      createTable("test", client);
-      long start = System.nanoTime();
-      for (int i = 0; i < 1000; i++) {
-        long now = System.nanoTime();
-        if (start + 5000000000L < now) {
-          System.out.println("Total [" + i + "]");
-          start = now;
-        }
-        addRow("test", i, client);
-      }
-
-      // This waits for all the data to become visible.
-      Thread.sleep(2000);
-
-      for (int i = 0; i < 1000; i++) {
-        searchRow("test", i, client);
-      }
-
-    } finally {
-      stopShards();
-      stopControllers();
-      shutdownZooKeeper();
-      shutdownDfs();
-    }
-  }
-
-  public static void startBlurCluster(String path, int controllerCount, int shardCount) {
-    startDfs(path);
-    startZooKeeper(path);
-    startControllers(controllerCount);
-    startShards(shardCount);
-  }
-
-  public static void shutdownBlurCluster() {
-    stopShards();
-    stopControllers();
-    shutdownZooKeeper();
-    shutdownDfs();
-  }
-
-  private static void createTable(String test, Iface client) throws BlurException, TException, IOException {
-    final TableDescriptor descriptor = new TableDescriptor();
-    descriptor.setName(test);
-    descriptor.setShardCount(7);
-    descriptor.setAnalyzerDefinition(new AnalyzerDefinition());
-    descriptor.setTableUri(getFileSystemUri() + "/blur/" + test);
-    client.createTable(descriptor);
-  }
-
-  public static String getControllerConnectionStr() {
-    return controllerConnectionStr;
-  }
-
-  private static void addRow(String table, int i, Iface client) throws BlurException, TException {
-    Row row = new Row();
-    row.setId(Integer.toString(i));
-    Record record = new Record();
-    record.setRecordId(Integer.toString(i));
-    record.setFamily("test");
-    record.addToColumns(new Column("test", Integer.toString(i)));
-    row.addToRecords(record);
-    RowMutation rowMutation = BlurUtil.toRowMutation(table, row);
-    rowMutation.setWal(false);
-    client.mutate(rowMutation);
-  }
-
-  private static void searchRow(String table, int i, Iface client) throws BlurException, TException {
-    BlurQuery blurQuery = BlurUtil.newSimpleQuery("test.test:" + i);
-    System.out.println("Running [" + blurQuery + "]");
-    BlurResults results = client.query(table, blurQuery);
-    if (results.getTotalResults() != 1L) {
-      throw new RuntimeException("we got a problem here.");
-    }
-  }
-
-  public static void stopControllers() {
-    for (ThriftServer s : controllers) {
-      s.close();
-    }
-  }
-
-  public static void stopShards() {
-    for (ThriftServer s : shards) {
-      s.close();
-    }
-  }
-
-  public static void startControllers(int num) {
-    BlurConfiguration configuration = getBlurConfiguration();
-    startControllers(configuration, num);
-  }
-
-  private static BlurConfiguration getBlurConfiguration() {
-    BlurConfiguration configuration;
-    try {
-      configuration = new BlurConfiguration();
-    } catch (IOException e) {
-      LOG.error(e);
-      throw new RuntimeException(e);
-    }
-    configuration.set(BLUR_ZOOKEEPER_CONNECTION, getZkConnectionString());
-    configuration.set(BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, "false");
-    configuration.set(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, "0");
-    configuration.setLong(BLUR_SHARD_SAFEMODEDELAY, 5000);
-    configuration.setInt(BLUR_GUI_CONTROLLER_PORT, -1);
-    configuration.setInt(BLUR_GUI_SHARD_PORT, -1);
-    return configuration;
-  }
-
-  public static void startControllers(BlurConfiguration configuration, int num) {
-    StringBuilder builder = new StringBuilder();
-    for (int i = 0; i < num; i++) {
-      try {
-        ThriftServer server = ThriftBlurControllerServer.createServer(i, configuration);
-        controllers.add(server);
-        Connection connection = new Connection("localhost", 40010 + i);
-        if (builder.length() != 0) {
-          builder.append(',');
-        }
-        builder.append(connection.getConnectionStr());
-        startServer(server, connection);
-      } catch (Exception e) {
-        LOG.error(e);
-        throw new RuntimeException(e);
-      }
-    }
-    controllerConnectionStr = builder.toString();
-  }
-
-  public static void startShards(int num) {
-    BlurConfiguration configuration = getBlurConfiguration();
-    startShards(configuration, num);
-  }
-
-  public static void startShards(final BlurConfiguration configuration, int num) {
-    ExecutorService executorService = Executors.newFixedThreadPool(num);
-    List<Future<ThriftServer>> futures = new ArrayList<Future<ThriftServer>>();
-    for (int i = 0; i < num; i++) {
-      final int index = i;
-      futures.add(executorService.submit(new Callable<ThriftServer>() {
-        @Override
-        public ThriftServer call() throws Exception {
-          return ThriftBlurShardServer.createServer(index, configuration);
-        }
-      }));
-    }
-
-    for (int i = 0; i < num; i++) {
-      try {
-        ThriftServer server = futures.get(i).get();
-        shards.add(server);
-        Connection connection = new Connection("localhost", 40020 + i);
-        startServer(server, connection);
-      } catch (Exception e) {
-        LOG.error(e);
-        throw new RuntimeException(e);
-      }
-    }
-  }
-
-  private static void startServer(final ThriftServer server, Connection connection) {
-    new Thread(new Runnable() {
-      @Override
-      public void run() {
-        try {
-          server.start();
-        } catch (TTransportException e) {
-          LOG.error(e);
-          throw new RuntimeException(e);
-        }
-      }
-    }).start();
-    while (true) {
-      try {
-        Thread.sleep(50);
-      } catch (InterruptedException e) {
-        return;
-      }
-      try {
-        Client client = BlurClientManager.newClient(connection);
-        BlurClientManager.close(client);
-        break;
-      } catch (TException e) {
-        throw new RuntimeException(e);
-      } catch (IOException e) {
-        LOG.info("Can not connection to [" + connection + "]");
-      }
-    }
-  }
-
-  public static String getZkConnectionString() {
-    return zkConnectionString;
-  }
-
-  public static void startZooKeeper(String path) {
-    startZooKeeper(true, path);
-  }
-
-  public static void startZooKeeper(boolean format, String path) {
-    Properties properties = new Properties();
-    properties.setProperty("tickTime", "2000");
-    properties.setProperty("initLimit", "10");
-    properties.setProperty("syncLimit", "5");
-
-    properties.setProperty("clientPort", "21810");
-
-    startZooKeeper(properties, format, path);
-  }
-
-  public static void startZooKeeper(Properties properties, String path) {
-    startZooKeeper(properties, true, path);
-  }
-
-  private static class ZooKeeperServerMainEmbedded extends ZooKeeperServerMain {
-    @Override
-    public void shutdown() {
-      super.shutdown();
-    }
-  }
-
-  public static void startZooKeeper(final Properties properties, boolean format, String path) {
-    String realPath = path + "/zk_test";
-    properties.setProperty("dataDir", realPath);
-    final ServerConfig serverConfig = new ServerConfig();
-    QuorumPeerConfig config = new QuorumPeerConfig();
-    try {
-      config.parseProperties(properties);
-    } catch (IOException e) {
-      LOG.error(e);
-      throw new RuntimeException(e);
-    } catch (ConfigException e) {
-      LOG.error(e);
-      throw new RuntimeException(e);
-    }
-    serverConfig.readFrom(config);
-    rm(new File(realPath));
-    serverThread = new Thread(new Runnable() {
-
-      @Override
-      public void run() {
-        try {
-          zooKeeperServerMain = new ZooKeeperServerMainEmbedded();
-          zooKeeperServerMain.runFromConfig(serverConfig);
-        } catch (IOException e) {
-          LOG.error(e);
-        }
-      }
-    });
-    serverThread.start();
-    long s = System.nanoTime();
-    while (s + 10000000000L > System.nanoTime()) {
-      try {
-        Thread.sleep(50);
-      } catch (InterruptedException e) {
-        LOG.error(e);
-        throw new RuntimeException(e);
-      }
-      try {
-        ZooKeeper zk = new ZooKeeper(zkConnectionString, 30000, new Watcher() {
-          @Override
-          public void process(WatchedEvent event) {
-
-          }
-        });
-        zk.close();
-        break;
-      } catch (IOException e) {
-        LOG.error(e);
-        throw new RuntimeException(e);
-      } catch (InterruptedException e) {
-        LOG.error(e);
-        throw new RuntimeException(e);
-      }
-    }
-  }
-
-  public static URI getFileSystemUri() throws IOException {
-    return cluster.getFileSystem().getUri();
-  }
-
-  public static void startDfs(String path) {
-    startDfs(true, path);
-  }
-
-  public static void startDfs(boolean format, String path) {
-    startDfs(new Configuration(), format, path);
-  }
-
-  public static void startDfs(Configuration conf, String path) {
-    startDfs(conf, true, path);
-  }
-
-  public static void startDfs(Configuration conf, boolean format, String path) {
-    System.setProperty("test.build.data", path);
-    try {
-      cluster = new MiniDFSCluster(conf, 1, true, (String[]) null);
-      cluster.waitActive();
-    } catch (Exception e) {
-      LOG.error("error opening file system", e);
-      throw new RuntimeException(e);
-    }
-  }
-
-  public static void shutdownZooKeeper() {
-    zooKeeperServerMain.shutdown();
-  }
-
-  public static void shutdownDfs() {
-    if (cluster != null) {
-      LOG.info("Shutting down Mini DFS ");
-      try {
-        cluster.shutdown();
-      } catch (Exception e) {
-        // / Can get a java.lang.reflect.UndeclaredThrowableException thrown
-        // here because of an InterruptedException. Don't let exceptions in
-        // here be cause of test failure.
-      }
-      try {
-        FileSystem fs = cluster.getFileSystem();
-        if (fs != null) {
-          LOG.info("Shutting down FileSystem");
-          fs.close();
-        }
-        FileSystem.closeAll();
-      } catch (IOException e) {
-        LOG.error("error closing file system", e);
-      }
-
-      // This has got to be one of the worst hacks I have ever had to do.
-      // This is needed to shutdown 2 thread pools that are not shutdown by
-      // themselves.
-      ThreadGroup threadGroup = Thread.currentThread().getThreadGroup();
-      Thread[] threads = new Thread[100];
-      int enumerate = threadGroup.enumerate(threads);
-      for (int i = 0; i < enumerate; i++) {
-        Thread thread = threads[i];
-        if (thread.getName().startsWith("pool")) {
-          while (thread.isAlive()) {
-            LOG.info("Stopping ThreadPoolExecutor [" + thread.getName() + "]");
-            Object target = getField(thread, "target");
-            if (target != null) {
-              ThreadPoolExecutor e = (ThreadPoolExecutor) getField(target, "this$0");
-              if (e != null) {
-                e.shutdownNow();
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-
-  private static Object getField(Object o, String fieldName) {
-    try {
-      Field field = o.getClass().getDeclaredField(fieldName);
-      field.setAccessible(true);
-      return field.get(o);
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  private static void rm(File file) {
-    if (!file.exists()) {
-      return;
-    }
-    if (file.isDirectory()) {
-      for (File f : file.listFiles()) {
-        rm(f);
-      }
-    }
-    file.delete();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/test/java/com/nearinfinity/blur/analysis/BlurAnalyzerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/com/nearinfinity/blur/analysis/BlurAnalyzerTest.java b/src/blur-core/src/test/java/com/nearinfinity/blur/analysis/BlurAnalyzerTest.java
deleted file mode 100644
index 54735fd..0000000
--- a/src/blur-core/src/test/java/com/nearinfinity/blur/analysis/BlurAnalyzerTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-package com.nearinfinity.blur.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.lucene.document.Field.Store;
-import org.junit.Test;
-
-import com.nearinfinity.blur.thrift.generated.AlternateColumnDefinition;
-import com.nearinfinity.blur.thrift.generated.AnalyzerDefinition;
-import com.nearinfinity.blur.thrift.generated.ColumnDefinition;
-import com.nearinfinity.blur.thrift.generated.ColumnFamilyDefinition;
-
-public class BlurAnalyzerTest {
-
-  private static final String STANDARD = "org.apache.lucene.analysis.standard.StandardAnalyzer";
-  private AnalyzerDefinition analyzerDefinition = getDef();
-
-  @Test
-  public void testToAndFromJSON() throws IOException {
-    BlurAnalyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    String json = analyzer.toJSON();
-    BlurAnalyzer analyzer2 = BlurAnalyzer.create(json);
-    assertEquals(analyzer.getAnalyzerDefinition(), analyzer2.getAnalyzerDefinition());
-  }
-
-  @Test
-  public void testStoringOfField() throws IOException {
-    BlurAnalyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    assertEquals(Store.NO, analyzer.getStore("b.c.sub1"));
-    assertEquals(Store.YES, analyzer.getStore("b.c"));
-  }
-
-  @Test
-  public void testGetSubFields() throws IOException {
-    BlurAnalyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    assertNull(analyzer.getSubIndexNames("b.d"));
-    Set<String> subIndexNames = analyzer.getSubIndexNames("b.c");
-    TreeSet<String> set = new TreeSet<String>();
-    set.add("b.c.sub1");
-    set.add("b.c.sub2");
-    assertEquals(set, subIndexNames);
-  }
-
-  @Test
-  public void testFullTextFields() throws IOException {
-    BlurAnalyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    assertTrue(analyzer.isFullTextField("a.b"));
-    assertFalse(analyzer.isFullTextField("a.d"));
-  }
-
-  private AnalyzerDefinition getDef() {
-
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition().setDefaultDefinition(new ColumnDefinition(STANDARD, false, null)).setFullTextAnalyzerClassName(STANDARD);
-    Map<String, ColumnFamilyDefinition> columnFamilyDefinitions = new HashMap<String, ColumnFamilyDefinition>();
-
-    ColumnFamilyDefinition aColumnFamilyDefinition = new ColumnFamilyDefinition();
-
-    Map<String, ColumnDefinition> aColumnDefinitions = new HashMap<String, ColumnDefinition>();
-    aColumnDefinitions.put("b", new ColumnDefinition(STANDARD, true, null));
-    aColumnFamilyDefinition.setColumnDefinitions(aColumnDefinitions);
-    columnFamilyDefinitions.put("a", aColumnFamilyDefinition);
-
-    Map<String, ColumnDefinition> bColumnDefinitions = new HashMap<String, ColumnDefinition>();
-    Map<String, AlternateColumnDefinition> alternates = new HashMap<String, AlternateColumnDefinition>();
-    alternates.put("sub1", new AlternateColumnDefinition(STANDARD));
-    alternates.put("sub2", new AlternateColumnDefinition(STANDARD));
-    bColumnDefinitions.put("c", new ColumnDefinition(STANDARD, true, alternates));
-    ColumnFamilyDefinition bColumnFamilyDefinition = new ColumnFamilyDefinition();
-    bColumnFamilyDefinition.setColumnDefinitions(bColumnDefinitions);
-    columnFamilyDefinitions.put("b", bColumnFamilyDefinition);
-
-    analyzerDefinition.setColumnFamilyDefinitions(columnFamilyDefinitions);
-    return analyzerDefinition;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/test/java/com/nearinfinity/blur/analysis/LongAnalyzerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/com/nearinfinity/blur/analysis/LongAnalyzerTest.java b/src/blur-core/src/test/java/com/nearinfinity/blur/analysis/LongAnalyzerTest.java
deleted file mode 100644
index 63692ab..0000000
--- a/src/blur-core/src/test/java/com/nearinfinity/blur/analysis/LongAnalyzerTest.java
+++ /dev/null
@@ -1,79 +0,0 @@
-package com.nearinfinity.blur.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.NumericRangeQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Version;
-import org.junit.Test;
-
-import com.nearinfinity.blur.index.IndexWriter;
-import com.nearinfinity.blur.thrift.generated.AnalyzerDefinition;
-import com.nearinfinity.blur.thrift.generated.ColumnDefinition;
-import com.nearinfinity.blur.thrift.generated.ColumnFamilyDefinition;
-
-public class LongAnalyzerTest {
-
-  @Test
-  public void testLongAnalyzer() throws IOException {
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
-    ColumnFamilyDefinition cfDef = new ColumnFamilyDefinition();
-    cfDef.putToColumnDefinitions("test", new ColumnDefinition("long", true, null));
-    analyzerDefinition.putToColumnFamilyDefinitions("test", cfDef);
-    BlurAnalyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-
-    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36, analyzer);
-    Directory dir = new RAMDirectory();
-    IndexWriter indexWriter = new IndexWriter(dir, conf);
-    for (int i = 0; i < 1000; i++) {
-      Document document = new Document();
-      String value = Long.toString(i);
-      document.add(new Field("test.test", value, Store.YES, Index.ANALYZED_NO_NORMS));
-      FieldConverterUtil.convert(document, analyzer);
-      indexWriter.addDocument(document);
-    }
-    indexWriter.close();
-
-    IndexSearcher searcher = new IndexSearcher(IndexReader.open(dir));
-    NumericRangeQuery<Long> query = NumericRangeQuery.newLongRange("test.test", 0L, 2L, true, true);
-    Query rewrite = searcher.rewrite(query);
-    TopDocs docs = searcher.search(rewrite, 100);
-    ScoreDoc[] scoreDocs = docs.scoreDocs;
-    assertEquals(3, docs.totalHits);
-    for (int i = 0; i < docs.totalHits; i++) {
-      Document document = searcher.doc(scoreDocs[i].doc);
-      assertTrue(Integer.parseInt(document.get("test.test")) < 3);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/test/java/com/nearinfinity/blur/lucene/search/FacetQueryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/com/nearinfinity/blur/lucene/search/FacetQueryTest.java b/src/blur-core/src/test/java/com/nearinfinity/blur/lucene/search/FacetQueryTest.java
deleted file mode 100644
index d6299b0..0000000
--- a/src/blur-core/src/test/java/com/nearinfinity/blur/lucene/search/FacetQueryTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-package com.nearinfinity.blur.lucene.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicLongArray;
-
-import org.apache.lucene.analysis.KeywordAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Version;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class FacetQueryTest {
-
-  private IndexReader reader;
-
-  @Before
-  public void setup() throws CorruptIndexException, LockObtainFailedException, IOException {
-    reader = createIndex();
-  }
-
-  @After
-  public void tearDown() {
-
-  }
-
-  @Test
-  public void testFacetQueryNoSuper() throws IOException {
-    BooleanQuery bq = new BooleanQuery();
-    bq.add(new TermQuery(new Term("f1", "value")), Occur.SHOULD);
-    bq.add(new TermQuery(new Term("f2", "v3")), Occur.SHOULD);
-
-    Query f1 = new TermQuery(new Term("f2", "v4"));
-
-    BooleanQuery f2 = new BooleanQuery();
-    f2.add(new TermQuery(new Term("f1", "value")), Occur.MUST);
-    f2.add(new TermQuery(new Term("f2", "v3")), Occur.MUST);
-
-    Query[] facets = new Query[] { f1, f2 };
-
-    AtomicLongArray counts = new AtomicLongArray(facets.length);
-    FacetQuery facetQuery = new FacetQuery(bq, facets, counts);
-
-    IndexSearcher indexSearcher = new IndexSearcher(reader);
-    TopDocs topDocs = indexSearcher.search(facetQuery, 10);
-
-    for (int i = 0; i < counts.length(); i++) {
-      System.out.println(counts.get(i));
-    }
-  }
-
-  private IndexReader createIndex() throws CorruptIndexException, LockObtainFailedException, IOException {
-    RAMDirectory directory = new RAMDirectory();
-    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_35, new KeywordAnalyzer());
-    IndexWriter writer = new IndexWriter(directory, conf);
-    for (int i = 0; i < 10; i++) {
-      Document document = new Document();
-      document.add(new Field("f1", "value", Store.YES, Index.NOT_ANALYZED_NO_NORMS));
-      document.add(new Field("f2", "v" + i, Store.YES, Index.NOT_ANALYZED_NO_NORMS));
-      writer.addDocument(document);
-    }
-    writer.close();
-    return IndexReader.open(directory);
-  }
-
-}


Mime
View raw message