incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [3/6] BLUR-ID-64-changed-parser
Date Thu, 11 Apr 2013 03:10:51 GMT
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index 0e41741..4170c53 100644
--- a/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/src/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -20,6 +20,7 @@ import static org.apache.blur.utils.BlurConstants.BLUR_SAHRD_INDEX_SIMILARITY;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_DELETION_POLICY_MAXAGE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_COMMITS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_TIME_BETWEEN_REFRESHS;
+import static org.apache.blur.utils.BlurConstants.SUPER;
 
 import java.util.Map;
 import java.util.Map.Entry;
@@ -29,7 +30,7 @@ import java.util.concurrent.TimeUnit;
 import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
-import org.apache.blur.lucene.search.ScoreType;
+import org.apache.blur.thrift.generated.ScoreType;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -90,7 +91,7 @@ public class TableContext {
     tableContext.tablePath = new Path(tableDescriptor.getTableUri());
     tableContext.walTablePath = new Path(tableContext.tablePath, LOGS);
     tableContext.analyzer = new BlurAnalyzer(tableDescriptor.getAnalyzerDefinition());
-//    tableContext.defaultFieldName = tableDescriptor.getDefaultFieldName();
+    tableContext.defaultFieldName = SUPER;
     tableContext.table = tableDescriptor.getName();
     tableContext.descriptor = tableDescriptor;
     tableContext.timeBetweenCommits = configuration.getLong(BLUR_SHARD_TIME_BETWEEN_COMMITS, 60000);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/src/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
index dc3ecff..52a4573 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
@@ -23,6 +23,7 @@ import org.apache.blur.BlurConfiguration;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.manager.clusterstatus.ClusterStatus;
+import org.apache.blur.server.TableContext;
 import org.apache.blur.thrift.BException;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.TableDescriptor;
@@ -57,6 +58,7 @@ public abstract class TableAdmin implements Iface {
   @Override
   public final void createTable(TableDescriptor tableDescriptor) throws BlurException, TException {
     try {
+      TableContext.clear();
       // @todo Remove this once issue #27 is resolved
       if (tableDescriptor.compressionBlockSize > 32768) {
         tableDescriptor.compressionBlockSize = 32768;
@@ -76,6 +78,7 @@ public abstract class TableAdmin implements Iface {
   @Override
   public final void disableTable(String table) throws BlurException, TException {
     try {
+      TableContext.clear();
       String cluster = _clusterStatus.getCluster(false, table);
       if (cluster == null) {
         throw new BlurException("Table [" + table + "] not found.", null);
@@ -111,6 +114,7 @@ public abstract class TableAdmin implements Iface {
   @Override
   public final void enableTable(String table) throws BlurException, TException {
     try {
+      TableContext.clear();
       String cluster = _clusterStatus.getCluster(false, table);
       if (cluster == null) {
         throw new BlurException("Table [" + table + "] not found.", null);
@@ -167,6 +171,7 @@ public abstract class TableAdmin implements Iface {
   @Override
   public final void removeTable(String table, boolean deleteIndexFiles) throws BlurException, TException {
     try {
+      TableContext.clear();
       String cluster = _clusterStatus.getCluster(false, table);
       if (cluster == null) {
         throw new BlurException("Table [" + table + "] not found.", null);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-core/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java b/src/blur-core/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java
deleted file mode 100644
index f8d4ffb..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/analysis/BlurAnalyzerTest.java
+++ /dev/null
@@ -1,133 +0,0 @@
-package org.apache.blur.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.blur.thrift.generated.AlternateColumnDefinition;
-import org.apache.blur.thrift.generated.AnalyzerDefinition;
-import org.apache.blur.thrift.generated.ColumnDefinition;
-import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
-import org.apache.lucene.document.Field.Store;
-import org.junit.Test;
-
-public class BlurAnalyzerTest {
-
-  private static final String STANDARD = "org.apache.lucene.analysis.standard.StandardAnalyzer";
-
-  @Test
-  public void testToAndFromJSONDef1() throws IOException {
-    BlurAnalyzer analyzer = new BlurAnalyzer(getDef1());
-    String json = analyzer.toJSON();
-    BlurAnalyzer analyzer2 = BlurAnalyzer.create(json);
-    assertEquals(analyzer.getAnalyzerDefinition(), analyzer2.getAnalyzerDefinition());
-  }
-
-  @Test
-  public void testStoringOfFieldDef1() throws IOException {
-    BlurAnalyzer analyzer = new BlurAnalyzer(getDef1());
-    assertEquals(Store.NO, analyzer.getStore("b.c.sub1"));
-    assertEquals(Store.YES, analyzer.getStore("b.c"));
-  }
-
-  @Test
-  public void testGetSubFieldsDef1() throws IOException {
-    BlurAnalyzer analyzer = new BlurAnalyzer(getDef1());
-    assertNull(analyzer.getSubIndexNames("b.d"));
-    Set<String> subIndexNames = analyzer.getSubIndexNames("b.c");
-    TreeSet<String> set = new TreeSet<String>();
-    set.add("b.c.sub1");
-    set.add("b.c.sub2");
-    assertEquals(set, subIndexNames);
-  }
-
-  @Test
-  public void testFullTextFieldsDef1() throws IOException {
-    BlurAnalyzer analyzer = new BlurAnalyzer(getDef1());
-    assertTrue(analyzer.isFullTextField("a.b"));
-    assertFalse(analyzer.isFullTextField("a.d"));
-  }
-
-  @Test
-  public void testToAndFromJSONDef2() throws IOException {
-    BlurAnalyzer analyzer = new BlurAnalyzer(getDef2());
-    String json = analyzer.toJSON();
-    BlurAnalyzer analyzer2 = BlurAnalyzer.create(json);
-    assertEquals(analyzer.getAnalyzerDefinition(), analyzer2.getAnalyzerDefinition());
-  }
-
-  @Test
-  public void testStoringOfFieldDef2() throws IOException {
-    BlurAnalyzer analyzer = new BlurAnalyzer(getDef2());
-    assertEquals(Store.YES, analyzer.getStore("a.b"));
-    assertEquals(Store.YES, analyzer.getStore("b.c"));
-  }
-
-  @Test
-  public void testGetSubFieldsDef2() throws IOException {
-    BlurAnalyzer analyzer = new BlurAnalyzer(getDef2());
-    assertNull(analyzer.getSubIndexNames("b.d"));
-  }
-
-  @Test
-  public void testFullTextFieldsDef2() throws IOException {
-    BlurAnalyzer analyzer = new BlurAnalyzer(getDef2());
-    assertTrue(analyzer.isFullTextField("a.b"));
-    assertFalse(analyzer.isFullTextField("d.a"));
-  }
-
-  private AnalyzerDefinition getDef1() {
-
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition().setDefaultDefinition(new ColumnDefinition(STANDARD, false, null)).setFullTextAnalyzerClassName(STANDARD);
-    Map<String, ColumnFamilyDefinition> columnFamilyDefinitions = new HashMap<String, ColumnFamilyDefinition>();
-
-    ColumnFamilyDefinition aColumnFamilyDefinition = new ColumnFamilyDefinition();
-
-    Map<String, ColumnDefinition> aColumnDefinitions = new HashMap<String, ColumnDefinition>();
-    aColumnDefinitions.put("b", new ColumnDefinition(STANDARD, true, null));
-    aColumnFamilyDefinition.setColumnDefinitions(aColumnDefinitions);
-    columnFamilyDefinitions.put("a", aColumnFamilyDefinition);
-
-    Map<String, ColumnDefinition> bColumnDefinitions = new HashMap<String, ColumnDefinition>();
-    Map<String, AlternateColumnDefinition> alternates = new HashMap<String, AlternateColumnDefinition>();
-    alternates.put("sub1", new AlternateColumnDefinition(STANDARD));
-    alternates.put("sub2", new AlternateColumnDefinition(STANDARD));
-    bColumnDefinitions.put("c", new ColumnDefinition(STANDARD, true, alternates));
-    ColumnFamilyDefinition bColumnFamilyDefinition = new ColumnFamilyDefinition();
-    bColumnFamilyDefinition.setColumnDefinitions(bColumnDefinitions);
-    columnFamilyDefinitions.put("b", bColumnFamilyDefinition);
-
-    analyzerDefinition.setColumnFamilyDefinitions(columnFamilyDefinitions);
-    return analyzerDefinition;
-  }
-
-  private AnalyzerDefinition getDef2() {
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition().setDefaultDefinition(new ColumnDefinition(STANDARD, false, null)).setFullTextAnalyzerClassName(STANDARD);
-    analyzerDefinition.putToColumnFamilyDefinitions("a", new ColumnFamilyDefinition().setDefaultDefinition(new ColumnDefinition(STANDARD, true, null)));
-    return analyzerDefinition;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-core/src/test/java/org/apache/blur/analysis/DoubleAnalyzerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/analysis/DoubleAnalyzerTest.java b/src/blur-core/src/test/java/org/apache/blur/analysis/DoubleAnalyzerTest.java
deleted file mode 100644
index 7573025..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/analysis/DoubleAnalyzerTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-package org.apache.blur.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.blur.index.IndexWriter;
-import org.apache.blur.thrift.generated.AnalyzerDefinition;
-import org.apache.blur.thrift.generated.ColumnDefinition;
-import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.NumericRangeQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Version;
-import org.junit.Test;
-
-public class DoubleAnalyzerTest {
-
-  @Test
-  public void testLongAnalyzer() throws IOException {
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
-    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
-    columnDefinitions.put("test", new ColumnDefinition("double", false, null));
-    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
-    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
-    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    runTestString(analyzer);
-  }
-
-  @Test
-  public void testLongAnalyzerDifferentStep() throws IOException {
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
-    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
-    columnDefinitions.put("test", new ColumnDefinition("double,4", false, null));
-    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
-    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
-    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    runTestString(analyzer);
-  }
-
-  private void runTestString(Analyzer analyzer) throws IOException {
-    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36, analyzer);
-    Directory dir = new RAMDirectory();
-    IndexWriter indexWriter = new IndexWriter(dir, conf);
-    for (int i = 0; i < 1000; i++) {
-      Document document = new Document();
-      String value = Double.toString(i);
-      document.add(new Field("test.test", value, Store.YES, Index.ANALYZED_NO_NORMS));
-      indexWriter.addDocument(document);
-    }
-    indexWriter.close();
-
-    IndexSearcher searcher = new IndexSearcher(IndexReader.open(dir));
-    NumericRangeQuery<Double> query = NumericRangeQuery.newDoubleRange("test.test", 0.0, 2.0, true, true);
-    Query rewrite = searcher.rewrite(query);
-    TopDocs docs = searcher.search(rewrite, 100);
-    ScoreDoc[] scoreDocs = docs.scoreDocs;
-    assertEquals(3, docs.totalHits);
-    for (int i = 0; i < docs.totalHits; i++) {
-      Document document = searcher.doc(scoreDocs[i].doc);
-      assertTrue(Double.parseDouble(document.get("test.test")) < 3.0);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-core/src/test/java/org/apache/blur/analysis/FloatAnalyzerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/analysis/FloatAnalyzerTest.java b/src/blur-core/src/test/java/org/apache/blur/analysis/FloatAnalyzerTest.java
deleted file mode 100644
index f989c90..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/analysis/FloatAnalyzerTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-package org.apache.blur.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.blur.index.IndexWriter;
-import org.apache.blur.thrift.generated.AnalyzerDefinition;
-import org.apache.blur.thrift.generated.ColumnDefinition;
-import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.NumericRangeQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Version;
-import org.junit.Test;
-
-public class FloatAnalyzerTest {
-
-  @Test
-  public void testLongAnalyzer() throws IOException {
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
-    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
-    columnDefinitions.put("test", new ColumnDefinition("float", false, null));
-    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
-    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
-    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    runTestString(analyzer);
-  }
-
-  @Test
-  public void testLongAnalyzerDifferentStep() throws IOException {
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
-    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
-    columnDefinitions.put("test", new ColumnDefinition("float,4", false, null));
-    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
-    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
-    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    runTestString(analyzer);
-  }
-
-  private void runTestString(Analyzer analyzer) throws IOException {
-    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36, analyzer);
-    Directory dir = new RAMDirectory();
-    IndexWriter indexWriter = new IndexWriter(dir, conf);
-    for (int i = 0; i < 1000; i++) {
-      Document document = new Document();
-      String value = Float.toString(i);
-      document.add(new Field("test.test", value, Store.YES, Index.ANALYZED_NO_NORMS));
-      indexWriter.addDocument(document);
-    }
-    indexWriter.close();
-
-    IndexSearcher searcher = new IndexSearcher(IndexReader.open(dir));
-    NumericRangeQuery<Float> query = NumericRangeQuery.newFloatRange("test.test", 0.0F, 2.0F, true, true);
-    Query rewrite = searcher.rewrite(query);
-    TopDocs docs = searcher.search(rewrite, 100);
-    ScoreDoc[] scoreDocs = docs.scoreDocs;
-    assertEquals(3, docs.totalHits);
-    for (int i = 0; i < docs.totalHits; i++) {
-      Document document = searcher.doc(scoreDocs[i].doc);
-      assertTrue(Float.parseFloat(document.get("test.test")) < 3.0F);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-core/src/test/java/org/apache/blur/analysis/IntegerAnalyzerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/analysis/IntegerAnalyzerTest.java b/src/blur-core/src/test/java/org/apache/blur/analysis/IntegerAnalyzerTest.java
deleted file mode 100644
index 6c28e6c..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/analysis/IntegerAnalyzerTest.java
+++ /dev/null
@@ -1,106 +0,0 @@
-package org.apache.blur.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.blur.index.IndexWriter;
-import org.apache.blur.thrift.generated.AnalyzerDefinition;
-import org.apache.blur.thrift.generated.ColumnDefinition;
-import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.NumericRangeQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Version;
-import org.junit.Test;
-
-public class IntegerAnalyzerTest {
-
-  @Test
-  public void testLongAnalyzer() throws IOException {
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
-    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
-    columnDefinitions.put("test", new ColumnDefinition("integer", false, null));
-    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
-    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
-    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    runTestString(analyzer, 10);
-  }
-
-  @Test
-  public void testLongAnalyzerDifferentStep() throws IOException {
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
-    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
-    columnDefinitions.put("test", new ColumnDefinition("integer,4", false, null));
-    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
-    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
-    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    runTestString(analyzer, 10);
-  }
-
-  @Test
-  public void testLongAnalyzerDifferentStepAndRadix() throws IOException {
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
-    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
-    columnDefinitions.put("test", new ColumnDefinition("integer,4,16", false, null));
-    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
-    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
-    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    runTestString(analyzer, 16);
-  }
-
-  private void runTestString(Analyzer analyzer, int radix) throws IOException {
-    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36, analyzer);
-    Directory dir = new RAMDirectory();
-    IndexWriter indexWriter = new IndexWriter(dir, conf);
-    for (int i = 0; i < 1000; i++) {
-      Document document = new Document();
-      String value = Integer.toString(i, radix);
-      document.add(new Field("test.test", value, Store.YES, Index.ANALYZED_NO_NORMS));
-      indexWriter.addDocument(document);
-    }
-    indexWriter.close();
-
-    IndexSearcher searcher = new IndexSearcher(IndexReader.open(dir));
-    NumericRangeQuery<Integer> query = NumericRangeQuery.newIntRange("test.test", 0, 2, true, true);
-    Query rewrite = searcher.rewrite(query);
-    TopDocs docs = searcher.search(rewrite, 100);
-    ScoreDoc[] scoreDocs = docs.scoreDocs;
-    assertEquals(3, docs.totalHits);
-    for (int i = 0; i < docs.totalHits; i++) {
-      Document document = searcher.doc(scoreDocs[i].doc);
-      assertTrue(Integer.parseInt(document.get("test.test"), radix) < 3);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-core/src/test/java/org/apache/blur/analysis/LongAnalyzerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/analysis/LongAnalyzerTest.java b/src/blur-core/src/test/java/org/apache/blur/analysis/LongAnalyzerTest.java
deleted file mode 100644
index c315933..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/analysis/LongAnalyzerTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-package org.apache.blur.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.blur.index.IndexWriter;
-import org.apache.blur.thrift.generated.AnalyzerDefinition;
-import org.apache.blur.thrift.generated.ColumnDefinition;
-import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.NumericRangeQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Version;
-import org.junit.Test;
-
-public class LongAnalyzerTest {
-
-  @Test
-  public void testLongAnalyzer() throws IOException {
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
-    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
-    columnDefinitions.put("test", new ColumnDefinition("long", false, null));
-    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
-    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
-    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    runTestString(analyzer, 10);
-  }
-
-  @Test
-  public void testLongAnalyzerDifferentStep() throws IOException {
-
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
-    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
-    columnDefinitions.put("test", new ColumnDefinition("long,4", false, null));
-    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
-    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
-    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    runTestString(analyzer, 10);
-  }
-
-  @Test
-  public void testLongAnalyzerDifferentStepAndRadix() throws IOException {
-    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
-    Map<String, ColumnDefinition> columnDefinitions = new HashMap<String, ColumnDefinition>();
-    columnDefinitions.put("test", new ColumnDefinition("long,4,16", false, null));
-    ColumnFamilyDefinition val = new ColumnFamilyDefinition(null, columnDefinitions);
-    analyzerDefinition.putToColumnFamilyDefinitions("test", val);
-    Analyzer analyzer = new BlurAnalyzer(analyzerDefinition);
-    runTestString(analyzer, 16);
-  }
-
-  private void runTestString(Analyzer analyzer, int radix) throws IOException {
-    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_36, analyzer);
-    Directory dir = new RAMDirectory();
-    IndexWriter indexWriter = new IndexWriter(dir, conf);
-    for (int i = 0; i < 1000; i++) {
-      Document document = new Document();
-      String value = Long.toString(i, radix);
-      document.add(new Field("test.test", value, Store.YES, Index.ANALYZED_NO_NORMS));
-      indexWriter.addDocument(document);
-    }
-    indexWriter.close();
-
-    IndexSearcher searcher = new IndexSearcher(IndexReader.open(dir));
-    NumericRangeQuery<Long> query = NumericRangeQuery.newLongRange("test.test", 0L, 2L, true, true);
-    Query rewrite = searcher.rewrite(query);
-    TopDocs docs = searcher.search(rewrite, 100);
-    ScoreDoc[] scoreDocs = docs.scoreDocs;
-    assertEquals(3, docs.totalHits);
-    for (int i = 0; i < docs.totalHits; i++) {
-      Document document = searcher.doc(scoreDocs[i].doc);
-      assertTrue(Long.parseLong(document.get("test.test"), radix) < 3);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-core/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java b/src/blur-core/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java
deleted file mode 100644
index fadd0d5..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-package org.apache.blur.lucene.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicLongArray;
-
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.store.RAMDirectory;
-import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-
-public class FacetQueryTest {
-
-  private IndexReader reader;
-
-  @Before
-  public void setup() throws CorruptIndexException, LockObtainFailedException, IOException {
-    reader = createIndex();
-  }
-
-  @After
-  public void tearDown() {
-
-  }
-
-  @Test
-  public void testFacetQueryNoSuper() throws IOException {
-    BooleanQuery bq = new BooleanQuery();
-    bq.add(new TermQuery(new Term("f1", "value")), Occur.SHOULD);
-    bq.add(new TermQuery(new Term("f2", "v3")), Occur.SHOULD);
-
-    Query f1 = new TermQuery(new Term("f2", "v4"));
-
-    BooleanQuery f2 = new BooleanQuery();
-    f2.add(new TermQuery(new Term("f1", "value")), Occur.MUST);
-    f2.add(new TermQuery(new Term("f2", "v3")), Occur.MUST);
-
-    Query[] facets = new Query[] { f1, f2 };
-
-    AtomicLongArray counts = new AtomicLongArray(facets.length);
-    FacetQuery facetQuery = new FacetQuery(bq, facets, counts);
-
-    IndexSearcher indexSearcher = new IndexSearcher(reader);
-    indexSearcher.search(facetQuery, 10);
-
-    //@TODO add actual assertion
-    for (int i = 0; i < counts.length(); i++) {
-      System.out.println(counts.get(i));
-    }
-  }
-
-  private IndexReader createIndex() throws CorruptIndexException, LockObtainFailedException, IOException {
-    RAMDirectory directory = new RAMDirectory();
-    IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
-    IndexWriter writer = new IndexWriter(directory, conf);
-    FieldType fieldType = new FieldType();
-    fieldType.setStored(true);
-    fieldType.setIndexed(true);
-    fieldType.setOmitNorms(true);
-    for (int i = 0; i < 10; i++) {
-      Document document = new Document();
-      
-      document.add(new Field("f1", "value", fieldType));
-      document.add(new Field("f2", "v" + i, fieldType));
-      writer.addDocument(document);
-    }
-    writer.close();
-    return DirectoryReader.open(directory);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-core/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java b/src/blur-core/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
deleted file mode 100644
index 498fb08..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/lucene/search/SuperParserTest.java
+++ /dev/null
@@ -1,236 +0,0 @@
-package org.apache.blur.lucene.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.List;
-
-import org.apache.blur.analysis.BlurAnalyzer;
-import org.apache.blur.thrift.generated.AnalyzerDefinition;
-import org.apache.blur.thrift.generated.ColumnDefinition;
-import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
-import org.apache.blur.thrift.generated.ScoreType;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.NumericRangeQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
-import org.junit.Before;
-import org.junit.Test;
-
-public class SuperParserTest {
-
-  private BlurAnalyzer analyzer;
-
-  @Before
-  public void setup() {
-    AnalyzerDefinition ad = new AnalyzerDefinition();
-    ad.setDefaultDefinition(new ColumnDefinition(StandardAnalyzer.class.getName(), true, null));
-    ColumnFamilyDefinition cfDef = new ColumnFamilyDefinition();
-    cfDef.putToColumnDefinitions("id_l", new ColumnDefinition("long", false, null));
-    cfDef.putToColumnDefinitions("id_d", new ColumnDefinition("double", false, null));
-    cfDef.putToColumnDefinitions("id_f", new ColumnDefinition("float", false, null));
-    cfDef.putToColumnDefinitions("id_i", new ColumnDefinition("integer", false, null));
-    ad.putToColumnFamilyDefinitions("a", cfDef);
-    analyzer = new BlurAnalyzer(ad);
-  }
-
-  @Test
-  public void test1() throws ParseException {
-    Query q = parseSq("(a.b:cool) (+a.c:cool a.b:cool)");
-    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(bq(bc_m(tq("a.c", "cool")), bc(tq("a.b", "cool")))))), q);
-  }
-
-  @Test
-  public void test2() throws ParseException {
-    Query q = parseSq("(a.b:cool) (a.c:cool a.b:cool)");
-    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(bq(bc(tq("a.c", "cool")), bc(tq("a.b", "cool")))))), q);
-  }
-
-  @Test
-  public void test3() throws ParseException {
-    Query q = parseSq("a.b:cool (a.c:cool a.b:cool)");
-    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(bq(bc(tq("a.c", "cool")), bc(tq("a.b", "cool")))))), q);
-  }
-
-  @Test
-  public void test4() throws ParseException {
-    Query q = parseSq("a.b:cool a.c:cool a.b:cool");
-    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(tq("a.c", "cool"))), bc(sq(tq("a.b", "cool")))), q);
-  }
-
-  @Test
-  public void test5() throws ParseException {
-    Query q = parseSq("(a.b:cool) (+a.c:cool c.b:cool)");
-    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(bq(bc_m(tq("a.c", "cool")), bc(tq("c.b", "cool")))))), q);
-  }
-
-  @Test
-  public void test6() throws ParseException {
-    Query q = parseSq("(a.b:cool) (a.c:cool c.b:cool)");
-    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(bq(bc(tq("a.c", "cool")), bc(tq("c.b", "cool")))))), q);
-  }
-
-  @Test
-  public void test7() throws ParseException {
-    Query q = parseSq("a.b:cool (a.c:cool c.b:cool)");
-    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(bq(bc(tq("a.c", "cool")), bc(tq("c.b", "cool")))))), q);
-  }
-
-  @Test
-  public void test8() throws ParseException {
-    Query q = parseSq("a.b:cool a.c:cool c.b:cool");
-    assertQuery(bq(bc(sq(tq("a.b", "cool"))), bc(sq(tq("a.c", "cool"))), bc(sq(tq("c.b", "cool")))), q);
-  }
-
-  @Test
-  public void test9() throws ParseException {
-    Query q = parseSq("a.id_l:[0 TO 2]");
-    assertQuery(sq(rq_i("a.id_l", 0L, 2L)), q);
-  }
-  
-  @Test
-  public void test10() throws ParseException {
-    Query q = parseSq("a.id_d:[0 TO 2]");
-    assertQuery(sq(rq_i("a.id_d", 0.0D, 2.0D)), q);
-  }
-  
-  @Test
-  public void test11() throws ParseException {
-    Query q = parseSq("a.id_f:[0 TO 2]");
-    assertQuery(sq(rq_i("a.id_f", 0.0F, 2.0F)), q);
-  }
-  
-  @Test
-  public void test12() throws ParseException {
-    Query q = parseSq("a.id_i:[0 TO 2]");
-    assertQuery(sq(rq_i("a.id_i", 0, 2)), q);
-  }
-
-  public static BooleanClause bc_m(Query q) {
-    return new BooleanClause(q, Occur.MUST);
-  }
-
-  public static BooleanClause bc_n(Query q) {
-    return new BooleanClause(q, Occur.MUST_NOT);
-  }
-
-  public static BooleanClause bc(Query q) {
-    return new BooleanClause(q, Occur.SHOULD);
-  }
-
-  public static void assertQuery(Query expected, Query actual) {
-    System.out.println(expected);
-    System.out.println(actual);
-    assertEqualsQuery(expected, actual);
-  }
-
-  public static void assertEqualsQuery(Query expected, Query actual) {
-    assertEquals(expected.getClass(), actual.getClass());
-    if (expected instanceof BooleanQuery) {
-      assertEqualsBooleanQuery((BooleanQuery) expected, (BooleanQuery) actual);
-    } else if (expected instanceof SuperQuery) {
-      assertEqualsSuperQuery((SuperQuery) expected, (SuperQuery) actual);
-    } else if (expected instanceof TermQuery) {
-      assertEqualsTermQuery((TermQuery) expected, (TermQuery) actual);
-    } else {
-      fail("Type [" + expected.getClass() + "] not supported");
-    }
-  }
-
-  public static void assertEqualsTermQuery(TermQuery expected, TermQuery actual) {
-    Term term1 = expected.getTerm();
-    Term term2 = actual.getTerm();
-    assertEquals(term1, term2);
-  }
-
-  public static void assertEqualsSuperQuery(SuperQuery expected, SuperQuery actual) {
-    assertEquals(expected.getQuery(), actual.getQuery());
-  }
-
-  public static void assertEqualsBooleanQuery(BooleanQuery expected, BooleanQuery actual) {
-    List<BooleanClause> clauses1 = expected.clauses();
-    List<BooleanClause> clauses2 = actual.clauses();
-    assertEqualsBooleanClause(clauses1, clauses2);
-  }
-
-  public static void assertEqualsBooleanClause(List<BooleanClause> clauses1, List<BooleanClause> clauses2) {
-    if (clauses1 == null && clauses2 == null) {
-      return;
-    }
-    if (clauses1 == null || clauses2 == null) {
-      fail();
-    }
-    if (clauses1.size() != clauses2.size()) {
-      fail();
-    }
-    int size = clauses1.size();
-    for (int i = 0; i < size; i++) {
-      assertEqualsBooleanClause(clauses1.get(i), clauses2.get(i));
-    }
-  }
-
-  public static void assertEqualsBooleanClause(BooleanClause booleanClause1, BooleanClause booleanClause2) {
-    assertEquals(booleanClause1.getOccur(), booleanClause2.getOccur());
-    assertEqualsQuery(booleanClause1.getQuery(), booleanClause2.getQuery());
-  }
-  
-  private Query rq_i(String field, float min, float max) {
-    return NumericRangeQuery.newFloatRange(field, min, max, true, true);
-  }
-  
-  private Query rq_i(String field, int min, int max) {
-    return NumericRangeQuery.newIntRange(field, min, max, true, true);
-  }
-  
-  private Query rq_i(String field, double min, double max) {
-    return NumericRangeQuery.newDoubleRange(field, min, max, true, true);
-  }
-
-  private Query rq_i(String field, long min, long max) {
-    return NumericRangeQuery.newLongRange(field, min, max, true, true);
-  }
-
-  private BooleanQuery bq(BooleanClause... bcs) {
-    BooleanQuery bq = new BooleanQuery();
-    for (BooleanClause bc : bcs) {
-      bq.add(bc);
-    }
-    return bq;
-  }
-
-  private SuperQuery sq(Query q) {
-    return new SuperQuery(q, ScoreType.SUPER);
-  }
-
-  private TermQuery tq(String field, String text) {
-    return new TermQuery(new Term(field, text));
-  }
-
-  private Query parseSq(String qstr) throws ParseException {
-    SuperParser parser = new SuperParser(LUCENE_VERSION, analyzer, true, null, ScoreType.SUPER);
-    return parser.parse(qstr);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java b/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
deleted file mode 100644
index c85eeeb..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
+++ /dev/null
@@ -1,157 +0,0 @@
-package org.apache.blur.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static junit.framework.Assert.assertTrue;
-import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Random;
-
-import org.apache.blur.analysis.BlurAnalyzer;
-import org.apache.blur.index.IndexWriter;
-import org.apache.blur.lucene.search.SuperParser;
-import org.apache.blur.thrift.generated.Column;
-import org.apache.blur.thrift.generated.Record;
-import org.apache.blur.thrift.generated.Row;
-import org.apache.blur.thrift.generated.ScoreType;
-import org.apache.blur.utils.RowIndexWriter;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.QueryWrapperFilter;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.store.RAMDirectory;
-import org.junit.Test;
-
-
-public class RandomSuperQueryTest {
-
-  private static final int MOD_COLS_USED_FOR_SKIPPING = 3;
-  private static final int MAX_NUM_OF_DOCS = 10000;// 10000
-  private static final int MIN_NUM_COL_FAM = 3;// 3
-  private static final int MAX_NUM_COL_FAM = 20;// 20
-  private static final int MAX_NUM_DOCS_PER_COL_FAM = 25;// 25
-  private static final int MAX_NUM_COLS = 21;// 21
-  private static final int MIN_NUM_COLS = 3;// 3
-  private static final int MAX_NUM_OF_WORDS = 1000;
-  private static final int MOD_USED_FOR_SAMPLING = 1;//
-
-  private Random seedGen = new Random(1);
-
-  @Test
-  public void testRandomSuperQuery() throws CorruptIndexException, IOException, InterruptedException, ParseException {
-    long seed = seedGen.nextLong();
-
-    Filter filter = new QueryWrapperFilter(new MatchAllDocsQuery());
-
-    Random random = new Random(seed);
-    Collection<String> sampler = new HashSet<String>();
-    System.out.print("Creating index... ");
-    System.out.flush();
-    Directory directory = createIndex(random, sampler);
-    IndexReader reader = DirectoryReader.open(directory);
-    System.out.print("Running searches [" + sampler.size() + "]... ");
-    System.out.flush();
-    assertTrue(!sampler.isEmpty());
-    IndexSearcher searcher = new IndexSearcher(reader);
-    long s = System.currentTimeMillis();
-    for (String str : sampler) {
-      Query query = new SuperParser(LUCENE_VERSION, new BlurAnalyzer(), true, filter, ScoreType.AGGREGATE).parse(str);
-      TopDocs topDocs = searcher.search(query, 10);
-      assertTrue("seed [" + seed + "] {" + query + "} {" + s + "}", topDocs.totalHits > 0);
-    }
-    long e = System.currentTimeMillis();
-    System.out.println("Finished in [" + (e - s) + "] ms");
-  }
-
-  private Directory createIndex(Random random, Collection<String> sampler) throws CorruptIndexException, LockObtainFailedException, IOException {
-    Directory directory = new RAMDirectory();
-    String[] columnFamilies = genWords(random, MIN_NUM_COL_FAM, MAX_NUM_COL_FAM, "colfam");
-    Map<String, String[]> columns = new HashMap<String, String[]>();
-    for (int i = 0; i < columnFamilies.length; i++) {
-      columns.put(columnFamilies[i], genWords(random, MIN_NUM_COLS, MAX_NUM_COLS, "col"));
-    }
-    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
-    RowIndexWriter indexWriter = new RowIndexWriter(writer, new BlurAnalyzer(new StandardAnalyzer(LUCENE_VERSION)));
-    int numberOfDocs = random.nextInt(MAX_NUM_OF_DOCS) + 1;
-    for (int i = 0; i < numberOfDocs; i++) {
-      indexWriter.replace(false, generatSuperDoc(random, columns, sampler));
-    }
-    writer.close();
-    return directory;
-  }
-
-  private String[] genWords(Random random, int min, int max, String prefix) {
-    int numberOfColFam = random.nextInt(max - min) + min;
-    String[] str = new String[numberOfColFam];
-    for (int i = 0; i < numberOfColFam; i++) {
-      str[i] = genWord(random, prefix);
-    }
-    return str;
-  }
-
-  private Row generatSuperDoc(Random random, Map<String, String[]> columns, Collection<String> sampler) {
-    Row row = new Row().setId(Long.toString(random.nextLong()));
-    StringBuilder builder = new StringBuilder();
-    for (String colFam : columns.keySet()) {
-      String[] cols = columns.get(colFam);
-      for (int i = 0; i < random.nextInt(MAX_NUM_DOCS_PER_COL_FAM); i++) {
-        Record record = new Record();
-        record.setFamily(colFam);
-        record.setRecordId(Long.toString(random.nextLong()));
-        int staringLength = builder.length();
-        for (String column : cols) {
-          if (random.nextInt() % MOD_COLS_USED_FOR_SKIPPING == 0) {
-            String word = genWord(random, "word");
-            record.addToColumns(new Column(column, word));
-            if (random.nextInt() % MOD_USED_FOR_SAMPLING == 0) {
-              builder.append(" +" + colFam + "." + column + ":" + word);
-            }
-          }
-        }
-        if (builder.length() != staringLength) {
-          builder.append(" nojoin.nojoin ");
-        }
-        row.addToRecords(record);
-      }
-    }
-    String string = builder.toString().trim();
-    if (!string.isEmpty()) {
-      sampler.add(string);
-    }
-    return row;
-  }
-
-  private String genWord(Random random, String prefix) {
-    return prefix + random.nextInt(MAX_NUM_OF_WORDS);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java b/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java
deleted file mode 100644
index 7043179..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java
+++ /dev/null
@@ -1,204 +0,0 @@
-package org.apache.blur.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static junit.framework.Assert.assertEquals;
-import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
-import static org.apache.blur.utils.BlurConstants.ROW_ID;
-import static org.apache.blur.utils.BlurUtil.newColumn;
-import static org.apache.blur.utils.BlurUtil.newRecord;
-import static org.apache.blur.utils.BlurUtil.newRow;
-
-import java.io.IOException;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLongArray;
-
-import org.apache.blur.analysis.BlurAnalyzer;
-import org.apache.blur.index.IndexWriter;
-import org.apache.blur.lucene.search.FacetQuery;
-import org.apache.blur.lucene.search.SuperQuery;
-import org.apache.blur.thrift.generated.ScoreType;
-import org.apache.blur.utils.RowIndexWriter;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.store.RAMDirectory;
-import org.junit.Test;
-
-
-public class SuperQueryTest {
-
-  private static final String PERSON_NAME = "person.name";
-  private static final String ADDRESS_STREET = "address.street";
-
-  private static final String STREET = "street";
-  private static final String ADDRESS = "address";
-  private static final String PERSON = "person";
-  private static final String NAME = "name";
-
-  private static final String NAME1 = "jon";
-  private static final String NAME2 = "jane";
-  private static final String STREET2 = "main st";
-  private static final String STREET1 = "main";
-
-  @Test
-  public void testSimpleSuperQuery() throws CorruptIndexException, IOException, InterruptedException {
-    BooleanQuery booleanQuery = new BooleanQuery();
-    booleanQuery.add(wrapSuper(new TermQuery(new Term(PERSON_NAME, NAME1))), Occur.MUST);
-    booleanQuery.add(wrapSuper(new TermQuery(new Term(ADDRESS_STREET, STREET1))), Occur.MUST);
-
-    Directory directory = createIndex();
-    IndexReader reader = DirectoryReader.open(directory);
-
-    IndexSearcher searcher = new IndexSearcher(reader);
-    TopDocs topDocs = searcher.search(booleanQuery, 10);
-    assertEquals(2, topDocs.totalHits);
-    assertEquals("1", searcher.doc(topDocs.scoreDocs[0].doc).get(ROW_ID));
-    assertEquals("3", searcher.doc(topDocs.scoreDocs[1].doc).get(ROW_ID));
-  }
-
-  @Test
-  public void testAggregateScoreTypes() throws Exception {
-    IndexSearcher searcher = createSearcher();
-    BooleanQuery booleanQuery = new BooleanQuery();
-    booleanQuery.add(wrapSuper(PERSON_NAME, NAME1, ScoreType.AGGREGATE), Occur.SHOULD);
-    booleanQuery.add(wrapSuper(ADDRESS_STREET, STREET1, ScoreType.AGGREGATE), Occur.MUST);
-    TopDocs topDocs = searcher.search(booleanQuery, 10);
-    printTopDocs(topDocs);
-    assertEquals(3, topDocs.totalHits);
-    assertEquals(3.30, topDocs.scoreDocs[0].score, 0.01);
-    assertEquals(2.20, topDocs.scoreDocs[1].score, 0.01);
-    assertEquals(0.55, topDocs.scoreDocs[2].score, 0.01);
-  }
-
-  @Test
-  public void testBestScoreTypes() throws Exception {
-    IndexSearcher searcher = createSearcher();
-    BooleanQuery booleanQuery = new BooleanQuery();
-    booleanQuery.add(wrapSuper(PERSON_NAME, NAME1, ScoreType.BEST), Occur.SHOULD);
-    booleanQuery.add(wrapSuper(ADDRESS_STREET, STREET1, ScoreType.BEST), Occur.MUST);
-    TopDocs topDocs = searcher.search(booleanQuery, 10);
-    assertEquals(3, topDocs.totalHits);
-    printTopDocs(topDocs);
-    assertEquals(2.20, topDocs.scoreDocs[0].score, 0.01);
-    assertEquals(2.20, topDocs.scoreDocs[1].score, 0.01);
-    assertEquals(0.55, topDocs.scoreDocs[2].score, 0.01);
-  }
-
-  private void printTopDocs(TopDocs topDocs) {
-    for (int i = 0; i < topDocs.totalHits; i++) {
-      System.out.println("doc " + i + " score " + topDocs.scoreDocs[i].score);
-    }
-
-  }
-
-  @Test
-  public void testConstantScoreTypes() throws Exception {
-    IndexSearcher searcher = createSearcher();
-    BooleanQuery booleanQuery = new BooleanQuery();
-    booleanQuery.add(wrapSuper(PERSON_NAME, NAME1, ScoreType.CONSTANT), Occur.SHOULD);
-    booleanQuery.add(wrapSuper(ADDRESS_STREET, STREET1, ScoreType.CONSTANT), Occur.MUST);
-    TopDocs topDocs = searcher.search(booleanQuery, 10);
-    assertEquals(3, topDocs.totalHits);
-    printTopDocs(topDocs);
-    assertEquals(2.0, topDocs.scoreDocs[0].score, 0.01);
-    assertEquals(2.0, topDocs.scoreDocs[1].score, 0.01);
-    assertEquals(0.5, topDocs.scoreDocs[2].score, 0.01);
-  }
-
-  @Test
-  public void testSuperScoreTypes() throws Exception {
-    IndexSearcher searcher = createSearcher();
-    BooleanQuery booleanQuery = new BooleanQuery();
-    booleanQuery.add(wrapSuper(PERSON_NAME, NAME1, ScoreType.SUPER), Occur.SHOULD);
-    booleanQuery.add(wrapSuper(ADDRESS_STREET, STREET1, ScoreType.SUPER), Occur.MUST);
-    TopDocs topDocs = searcher.search(booleanQuery, 10);
-    assertEquals(3, topDocs.totalHits);
-    printTopDocs(topDocs);
-    assertEquals(3.10, topDocs.scoreDocs[0].score, 0.01);
-    assertEquals(3.00, topDocs.scoreDocs[1].score, 0.01);
-    assertEquals(0.75, topDocs.scoreDocs[2].score, 0.01);
-  }
-
-  @Test
-  public void testSuperScoreTypesWithFacet() throws Exception {
-    IndexSearcher searcher = createSearcher();
-    BooleanQuery booleanQuery = new BooleanQuery();
-    booleanQuery.add(wrapSuper(PERSON_NAME, NAME1, ScoreType.SUPER), Occur.SHOULD);
-    booleanQuery.add(wrapSuper(ADDRESS_STREET, STREET1, ScoreType.SUPER), Occur.MUST);
-
-    BooleanQuery f1 = new BooleanQuery();
-    f1.add(new TermQuery(new Term(PERSON_NAME, NAME1)), Occur.MUST);
-    f1.add(new TermQuery(new Term(PERSON_NAME, NAME2)), Occur.MUST);
-
-    Query[] facets = new Query[] { new SuperQuery(f1, ScoreType.CONSTANT) };
-    AtomicLongArray counts = new AtomicLongArray(facets.length);
-    FacetQuery query = new FacetQuery(booleanQuery, facets, counts);
-
-    TopDocs topDocs = searcher.search(query, 10);
-    assertEquals(3, topDocs.totalHits);
-    printTopDocs(topDocs);
-    assertEquals(3.10, topDocs.scoreDocs[0].score, 0.01);
-    assertEquals(3.00, topDocs.scoreDocs[1].score, 0.01);
-    assertEquals(0.75, topDocs.scoreDocs[2].score, 0.01);
-  }
-
-  private static IndexSearcher createSearcher() throws Exception {
-    Directory directory = createIndex();
-    IndexReader reader = DirectoryReader.open(directory);
-    return new IndexSearcher(reader);
-  }
-
-  public static Directory createIndex() throws CorruptIndexException, LockObtainFailedException, IOException {
-    Directory directory = new RAMDirectory();
-    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
-    BlurAnalyzer analyzer = new BlurAnalyzer(new StandardAnalyzer(LUCENE_VERSION));
-    RowIndexWriter indexWriter = new RowIndexWriter(writer, analyzer);
-    indexWriter.replace(
-        false,
-        newRow("1", newRecord(PERSON, UUID.randomUUID().toString(), newColumn(NAME, NAME1)), newRecord(PERSON, UUID.randomUUID().toString(), newColumn(NAME, NAME1)),
-            newRecord(ADDRESS, UUID.randomUUID().toString(), newColumn(STREET, STREET1))));
-    indexWriter.replace(false,
-        newRow("2", newRecord(PERSON, UUID.randomUUID().toString(), newColumn(NAME, NAME2)), newRecord(ADDRESS, UUID.randomUUID().toString(), newColumn(STREET, STREET1))));
-    indexWriter.replace(false,
-        newRow("3", newRecord(PERSON, UUID.randomUUID().toString(), newColumn(NAME, NAME1)), newRecord(ADDRESS, UUID.randomUUID().toString(), newColumn(STREET, STREET2))));
-    ;
-    writer.close();
-    return directory;
-  }
-
-  private Query wrapSuper(Query query) {
-    return new SuperQuery(query, ScoreType.AGGREGATE);
-  }
-
-  private Query wrapSuper(String field, String value, ScoreType scoreType) {
-    return new SuperQuery(new TermQuery(new Term(field, value)), scoreType);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-core/src/test/java/org/apache/blur/search/TestingPagingCollector.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/search/TestingPagingCollector.java b/src/blur-core/src/test/java/org/apache/blur/search/TestingPagingCollector.java
deleted file mode 100644
index 5eaad3f..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/search/TestingPagingCollector.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package org.apache.blur.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.blur.lucene.search.IterablePaging;
-import org.apache.blur.lucene.search.IterablePaging.ProgressRef;
-import org.apache.blur.lucene.search.IterablePaging.TotalHitsRef;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.RAMDirectory;
-import org.junit.Test;
-
-/**
- * Testing the paging collector.
- * 
- * @author Aaron McCurry
- */
-public class TestingPagingCollector {
-
-  @Test
-  public void testNothingYet() {
-
-  }
-
-  public static void main(String[] args) throws Exception {
-    IndexReader reader = getReaderFlatScore(13245);
-    IndexSearcher searcher = new IndexSearcher(reader);
-
-    TotalHitsRef totalHitsRef = new TotalHitsRef();
-    ProgressRef progressRef = new ProgressRef();
-
-    TermQuery query = new TermQuery(new Term("f1", "value"));
-    IterablePaging paging = new IterablePaging(new AtomicBoolean(true), searcher, query, 100, null, null);
-
-    for (ScoreDoc sd : paging.skipTo(90).gather(20).totalHits(totalHitsRef).progress(progressRef)) {
-
-      System.out.println("time [" + progressRef.queryTime() + "] " + "total hits [" + totalHitsRef.totalHits() + "] "
-          + "searches [" + progressRef.searchesPerformed() + "] " + "position [" + progressRef.currentHitPosition()
-          + "] " + "doc id [" + sd.doc + "] " + "score [" + sd.score + "]");
-    }
-  }
-
-  private static IndexReader getReaderFlatScore(int length) throws Exception {
-    RAMDirectory directory = new RAMDirectory();
-    IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer()));
-    for (int i = 0; i < length; i++) {
-      Document document = new Document();
-      document.add(new Field("f1", "value", Store.NO, Index.ANALYZED_NO_NORMS));
-      indexWriter.addDocument(document);
-    }
-    indexWriter.close();
-    return IndexReader.open(directory);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java b/src/blur-query/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
new file mode 100644
index 0000000..5b81291
--- /dev/null
+++ b/src/blur-query/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
@@ -0,0 +1,437 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
+import static org.apache.blur.util.BlurConstants.PRIME_DOC;
+import static org.apache.blur.util.BlurConstants.RECORD_ID;
+import static org.apache.blur.util.BlurConstants.ROW_ID;
+import static org.apache.blur.util.BlurConstants.SUPER;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.Constructor;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.blur.thrift.generated.AlternateColumnDefinition;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.ColumnDefinition;
+import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.AnalyzerWrapper;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.Version;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TJSONProtocol;
+import org.apache.thrift.transport.TMemoryBuffer;
+import org.apache.thrift.transport.TMemoryInputTransport;
+
+public final class BlurAnalyzer extends AnalyzerWrapper {
+
+  public enum TYPE {
+    LONG, DOUBLE, FLOAT, INTEGER, DEFAULT
+  }
+
+  @SuppressWarnings("serial")
+  private static Set<String> typeNameCache = new HashSet<String>() {
+    {
+      TYPE[] values = TYPE.values();
+      for (TYPE t : values) {
+        add(t.name());
+      }
+    }
+  };
+
+  private static final String STANDARD = "org.apache.lucene.analysis.standard.StandardAnalyzer";
+  public static final BlurAnalyzer BLANK_ANALYZER = new BlurAnalyzer(new KeywordAnalyzer());
+  private static Map<String, Class<? extends Analyzer>> aliases = new HashMap<String, Class<? extends Analyzer>>();
+
+  private Map<String, Store> _storeMap = new HashMap<String, Store>();
+  private Map<String, Set<String>> _subIndexNameLookups = new HashMap<String, Set<String>>();
+  private Map<String, Boolean> _fullTextFields = new HashMap<String, Boolean>();
+  private Map<String, Boolean> _fullTextColumnFamilies = new HashMap<String, Boolean>();
+  private AnalyzerDefinition _analyzerDefinition;
+  private Analyzer _fullTextAnalyzer = new StandardAnalyzer(LUCENE_VERSION);
+  private Analyzer _defaultAnalyzer;
+  private Map<String, Analyzer> _analyzers = new HashMap<String, Analyzer>();
+  private Map<String, TYPE> _typeLookup = new HashMap<String, BlurAnalyzer.TYPE>();
+
+  public Set<String> getSubIndexNames(String indexName) {
+    return _subIndexNameLookups.get(indexName);
+  }
+
+  public BlurAnalyzer(Analyzer analyzer) {
+    _analyzerDefinition = new AnalyzerDefinition();
+    _defaultAnalyzer = analyzer;
+  }
+
+  public BlurAnalyzer(AnalyzerDefinition analyzerDefinition) {
+    _analyzerDefinition = analyzerDefinition;
+    ColumnDefinition defaultDefinition = analyzerDefinition.getDefaultDefinition();
+    String fullTextAnalyzerClassName = analyzerDefinition.fullTextAnalyzerClassName;
+    if (fullTextAnalyzerClassName != null) {
+      _fullTextAnalyzer = getAnalyzerByClassName(fullTextAnalyzerClassName, aliases, null, null);
+    }
+    if (defaultDefinition == null) {
+      defaultDefinition = new ColumnDefinition(STANDARD, true, null);
+      analyzerDefinition.setDefaultDefinition(defaultDefinition);
+    }
+    _defaultAnalyzer = getAnalyzerByClassName(defaultDefinition.getAnalyzerClassName(), aliases, null, null);
+    KeywordAnalyzer keywordAnalyzer = new KeywordAnalyzer();
+    _analyzers = new HashMap<String, Analyzer>();
+    _analyzers.put(ROW_ID, keywordAnalyzer);
+    _analyzers.put(RECORD_ID, keywordAnalyzer);
+    _analyzers.put(PRIME_DOC, keywordAnalyzer);
+    _analyzers.put(SUPER, _fullTextAnalyzer);
+    load(_analyzers, _analyzerDefinition.columnFamilyDefinitions, _fullTextFields, _subIndexNameLookups, _storeMap, _fullTextColumnFamilies, _typeLookup);
+  }
+
+  public BlurAnalyzer() {
+    this(new StandardAnalyzer(LUCENE_VERSION));
+  }
+
+  private Analyzer getAnalyzer(String name) {
+    Analyzer analyzer = _analyzers.get(name);
+    return analyzer;
+  }
+
+  public TYPE getTypeLookup(String field) {
+    TYPE type = _typeLookup.get(field);
+    if (type == null) {
+      return TYPE.DEFAULT;
+    }
+    return type;
+  }
+
+  public Query getNewRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
+    Analyzer analyzer = getAnalyzer(field);
+    if (analyzer instanceof LongAnalyzer) {
+      LongAnalyzer a = (LongAnalyzer) analyzer;
+      int precisionStep = a.getPrecisionStep();
+      int radix = a.getRadix();
+      long min = Long.parseLong(part1, radix);
+      long max = Long.parseLong(part2, radix);
+      return NumericRangeQuery.newLongRange(field, precisionStep, min, max, startInclusive, endInclusive);
+    } else if (analyzer instanceof DoubleAnalyzer) {
+      DoubleAnalyzer a = (DoubleAnalyzer) analyzer;
+      int precisionStep = a.getPrecisionStep();
+      double min = Double.parseDouble(part1);
+      double max = Double.parseDouble(part2);
+      return NumericRangeQuery.newDoubleRange(field, precisionStep, min, max, startInclusive, endInclusive);
+    } else if (analyzer instanceof FloatAnalyzer) {
+      FloatAnalyzer a = (FloatAnalyzer) analyzer;
+      int precisionStep = a.getPrecisionStep();
+      float min = Float.parseFloat(part1);
+      float max = Float.parseFloat(part2);
+      return NumericRangeQuery.newFloatRange(field, precisionStep, min, max, startInclusive, endInclusive);
+    } else if (analyzer instanceof IntegerAnalyzer) {
+      IntegerAnalyzer a = (IntegerAnalyzer) analyzer;
+      int precisionStep = a.getPrecisionStep();
+      int radix = a.getRadix();
+      int min = Integer.parseInt(part1, radix);
+      int max = Integer.parseInt(part2, radix);
+      return NumericRangeQuery.newIntRange(field, precisionStep, min, max, startInclusive, endInclusive);
+    }
+    return null;
+  }
+
+  public boolean isFullTextField(String fieldName) {
+    Boolean b = _fullTextFields.get(fieldName);
+    if (b != null) {
+      return b;
+    }
+    String cf = getColumnFamily(fieldName);
+    if (cf == null) {
+      return false;
+    }
+    b = _fullTextColumnFamilies.get(cf);
+    if (b != null) {
+      return b;
+    }
+    ColumnDefinition defaultDefinition = _analyzerDefinition.getDefaultDefinition();
+    if (defaultDefinition != null && defaultDefinition.fullTextIndex) {
+      return true;
+    }
+    return false;
+  }
+
+  public Store getStore(String indexName) {
+    Store store = _storeMap.get(indexName);
+    if (store == null) {
+      return Store.YES;
+    }
+    return store;
+  }
+
+  public Index getIndex(String indexName) {
+    return Index.ANALYZED_NO_NORMS;
+  }
+
+  public String toJSON() {
+    TMemoryBuffer trans = new TMemoryBuffer(1024);
+    TJSONProtocol protocol = new TJSONProtocol(trans);
+    try {
+      _analyzerDefinition.write(protocol);
+    } catch (TException e) {
+      throw new RuntimeException(e);
+    }
+    trans.close();
+    byte[] array = trans.getArray();
+    return new String(array, 0, trans.length());
+  }
+
+  private String getColumnFamily(String fieldName) {
+    int index = fieldName.indexOf('.');
+    if (index < 0) {
+      return null;
+    }
+    return fieldName.substring(0, index);
+  }
+
+  public AnalyzerDefinition getAnalyzerDefinition() {
+    return _analyzerDefinition;
+  }
+
+  public void close() {
+
+  }
+  
+  @Override
+  protected Analyzer getWrappedAnalyzer(String fieldName) {
+    Analyzer analyzer = getAnalyzer(fieldName);
+    return (analyzer != null) ? analyzer : _defaultAnalyzer;
+  }
+
+  @Override
+  protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
+    return components;
+  }
+
+  public static BlurAnalyzer create(File file) throws IOException {
+    FileInputStream inputStream = new FileInputStream(file);
+    try {
+      return create(inputStream);
+    } finally {
+      inputStream.close();
+    }
+  }
+
+  public static BlurAnalyzer create(InputStream inputStream) throws IOException {
+    TMemoryInputTransport trans = new TMemoryInputTransport(getBytes(inputStream));
+    TJSONProtocol protocol = new TJSONProtocol(trans);
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
+    try {
+      analyzerDefinition.read(protocol);
+    } catch (TException e) {
+      throw new RuntimeException(e);
+    }
+    trans.close();
+    return new BlurAnalyzer(analyzerDefinition);
+  }
+
+  public static BlurAnalyzer create(String jsonStr) throws IOException {
+    InputStream inputStream = new ByteArrayInputStream(jsonStr.getBytes());
+    try {
+      return create(inputStream);
+    } finally {
+      inputStream.close();
+    }
+  }
+
+  public static BlurAnalyzer create(Path path) throws IOException {
+    FileSystem fileSystem = FileSystem.get(path.toUri(), new Configuration());
+    FSDataInputStream inputStream = fileSystem.open(path);
+    try {
+      return create(inputStream);
+    } finally {
+      inputStream.close();
+    }
+  }
+
+  private static byte[] getBytes(InputStream inputStream) throws IOException {
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    byte[] buffer = new byte[1024];
+    int num;
+    while ((num = inputStream.read(buffer)) != -1) {
+      outputStream.write(buffer, 0, num);
+    }
+    inputStream.close();
+    outputStream.close();
+    return outputStream.toByteArray();
+  }
+
+  private static void load(Map<String, Analyzer> analyzers, Map<String, ColumnFamilyDefinition> familyDefinitions, Map<String, Boolean> fullTextFields,
+      Map<String, Set<String>> subIndexNameLookups, Map<String, Store> storeMap, Map<String, Boolean> fullTextColumnFamilies, Map<String, TYPE> typeLookup) {
+    if (familyDefinitions != null) {
+      for (String family : familyDefinitions.keySet()) {
+        ColumnFamilyDefinition familyDefinition = familyDefinitions.get(family);
+        ColumnDefinition defaultDefinition = familyDefinition.getDefaultDefinition();
+        if (defaultDefinition != null) {
+          fullTextColumnFamilies.put(family, defaultDefinition.isFullTextIndex());
+        }
+        load(family, familyDefinition, analyzers, fullTextFields, subIndexNameLookups, storeMap, typeLookup);
+      }
+    }
+  }
+
+  private static void load(String family, ColumnFamilyDefinition familyDefinition, Map<String, Analyzer> analyzers, Map<String, Boolean> fullTextFields,
+      Map<String, Set<String>> subIndexNameLookups, Map<String, Store> storeMap, Map<String, TYPE> typeLookup) {
+    Map<String, ColumnDefinition> columnDefinitions = familyDefinition.getColumnDefinitions();
+    if (columnDefinitions != null) {
+      for (String column : columnDefinitions.keySet()) {
+        ColumnDefinition columnDefinition = columnDefinitions.get(column);
+        load(family, familyDefinition, column, columnDefinition, analyzers, fullTextFields, subIndexNameLookups, storeMap, typeLookup);
+      }
+    }
+  }
+
+  private static void load(String family, ColumnFamilyDefinition familyDefinition, String column, ColumnDefinition columnDefinition, Map<String, Analyzer> analyzers,
+      Map<String, Boolean> fullTextFields, Map<String, Set<String>> subIndexNameLookups, Map<String, Store> storeMap, Map<String, TYPE> typeLookup) {
+    Map<String, AlternateColumnDefinition> alternateColumnDefinitions = columnDefinition.getAlternateColumnDefinitions();
+    if (alternateColumnDefinitions != null) {
+      for (String subColumn : alternateColumnDefinitions.keySet()) {
+        AlternateColumnDefinition alternateColumnDefinition = alternateColumnDefinitions.get(subColumn);
+        load(family, familyDefinition, column, columnDefinition, subColumn, alternateColumnDefinition, analyzers, subIndexNameLookups, storeMap, typeLookup);
+      }
+    }
+    String fieldName = family + "." + column;
+    Analyzer analyzer = getAnalyzerByClassName(columnDefinition.getAnalyzerClassName(), aliases, fieldName, typeLookup);
+    analyzers.put(fieldName, analyzer);
+    if (columnDefinition.isFullTextIndex()) {
+      fullTextFields.put(fieldName, Boolean.TRUE);
+    } else {
+      fullTextFields.put(fieldName, Boolean.FALSE);
+    }
+  }
+
+  private static void load(String family, ColumnFamilyDefinition familyDefinition, String column, ColumnDefinition columnDefinition, String subColumn,
+      AlternateColumnDefinition alternateColumnDefinition, Map<String, Analyzer> analyzers, Map<String, Set<String>> subIndexNameLookups, Map<String, Store> storeMap,
+      Map<String, TYPE> typeLookup) {
+    String fieldName = family + "." + column + "." + subColumn;
+    Analyzer analyzer = getAnalyzerByClassName(alternateColumnDefinition.getAnalyzerClassName(), aliases, fieldName, typeLookup);
+    analyzers.put(fieldName, analyzer);
+    putStore(fieldName, Store.NO, storeMap);
+    addSubField(fieldName, subIndexNameLookups);
+  }
+
+  private static void putStore(String name, Store store, Map<String, Store> storeMap) {
+    storeMap.put(name, store);
+  }
+
+  @SuppressWarnings("unchecked")
+  private static Analyzer getAnalyzerByClassName(String className, Map<String, Class<? extends Analyzer>> aliases, String fieldName, Map<String, TYPE> typeLookup) {
+    Analyzer type = getType(className, fieldName, typeLookup);
+    if (type != null) {
+      return type;
+    }
+    try {
+      Class<? extends Analyzer> clazz = aliases.get(className);
+      if (clazz == null) {
+        clazz = (Class<? extends Analyzer>) Class.forName(className);
+      }
+      try {
+        return (Analyzer) clazz.newInstance();
+      } catch (Exception e) {
+        Constructor<?> constructor = clazz.getConstructor(new Class[] { Version.class });
+        return (Analyzer) constructor.newInstance(LUCENE_VERSION);
+      }
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private static Analyzer getType(String typeStr, String fieldName, Map<String, TYPE> typeLookup) {
+    if (typeStr == null) {
+      return null;
+    }
+    String[] types = typeStr.split(",");
+    String type = types[0];
+    String typeUpper = type.toUpperCase();
+    if (!typeNameCache.contains(typeUpper)) {
+      return null;
+    }
+    TYPE t = TYPE.valueOf(typeUpper);
+    if (fieldName != null) {
+      typeLookup.put(fieldName, t);
+    }
+    switch (t) {
+    case LONG:
+      LongAnalyzer longAnalyzer = new LongAnalyzer();
+      if (types.length > 1) {
+        longAnalyzer.setPrecisionStep(Integer.parseInt(types[1]));
+      }
+      if (types.length > 2) {
+        longAnalyzer.setRadix(Integer.parseInt(types[2]));
+      }
+      return longAnalyzer;
+    case INTEGER:
+      IntegerAnalyzer integerAnalyzer = new IntegerAnalyzer();
+      if (types.length > 1) {
+        integerAnalyzer.setPrecisionStep(Integer.parseInt(types[1]));
+      }
+      if (types.length > 2) {
+        integerAnalyzer.setRadix(Integer.parseInt(types[2]));
+      }
+      return integerAnalyzer;
+    case FLOAT:
+      FloatAnalyzer floatAnalyzer = new FloatAnalyzer();
+      if (types.length > 1) {
+        floatAnalyzer.setPrecisionStep(Integer.parseInt(types[1]));
+      }
+      return floatAnalyzer;
+    case DOUBLE:
+      DoubleAnalyzer doubleAnalyzer = new DoubleAnalyzer();
+      if (types.length > 1) {
+        doubleAnalyzer.setPrecisionStep(Integer.parseInt(types[1]));
+      }
+      return doubleAnalyzer;
+    default:
+      break;
+    }
+    return null;
+  }
+
+  private static void addSubField(String name, Map<String, Set<String>> subIndexNameLookups) {
+    int lastIndexOf = name.lastIndexOf('.');
+    String mainFieldName = name.substring(0, lastIndexOf);
+    Set<String> set = subIndexNameLookups.get(mainFieldName);
+    if (set == null) {
+      set = new TreeSet<String>();
+      subIndexNameLookups.put(mainFieldName, set);
+    }
+    set.add(name);
+  }
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c1231e34/src/blur-query/src/main/java/org/apache/blur/analysis/DoubleAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/main/java/org/apache/blur/analysis/DoubleAnalyzer.java b/src/blur-query/src/main/java/org/apache/blur/analysis/DoubleAnalyzer.java
new file mode 100644
index 0000000..01abeae
--- /dev/null
+++ b/src/blur-query/src/main/java/org/apache/blur/analysis/DoubleAnalyzer.java
@@ -0,0 +1,83 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.io.Reader;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.util.CharTokenizer;
+import org.apache.lucene.util.NumericUtils;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
+
+public final class DoubleAnalyzer extends Analyzer {
+
+  public static int PRECISION_STEP_DEFAULT = NumericUtils.PRECISION_STEP_DEFAULT;
+  private int precisionStep;
+
+  public DoubleAnalyzer() {
+    this(PRECISION_STEP_DEFAULT);
+  }
+
+  public DoubleAnalyzer(int precisionStep) {
+    this.precisionStep = precisionStep;
+  }
+
+  public int getPrecisionStep() {
+    return precisionStep;
+  }
+
+  public void setPrecisionStep(int precisionStep) {
+    this.precisionStep = precisionStep;
+  }
+  @Override
+  protected TokenStreamComponents createComponents(String fieldName,
+      Reader reader) {
+    final CharTokenizer source = new CharTokenizer(LUCENE_VERSION, reader) {
+      @Override
+      protected boolean isTokenChar(int arg0) {
+        return true;
+      }
+    };
+
+    final double value;
+    try {
+      value = toDouble(reader);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    final NumericTokenStream numericTokenStream = new NumericTokenStream(
+        precisionStep);
+    numericTokenStream.setDoubleValue(value);
+
+    return new TokenStreamComponents(source, numericTokenStream) {
+      public void setReader(Reader reader) throws IOException {
+        numericTokenStream.reset();
+        numericTokenStream.setDoubleValue(toDouble(reader));
+      }
+    };
+  }
+  private double toDouble(Reader reader) throws IOException {
+    StringBuilder builder = new StringBuilder(20);
+    int read;
+    while ((read = reader.read()) != -1) {
+      builder.append((char) read);
+    }
+    return Double.parseDouble(builder.toString());
+  }
+
+}


Mime
View raw message