incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/4] git commit: All unit test for core projects now pass.
Date Sun, 02 Dec 2012 04:23:16 GMT
Updated Branches:
  refs/heads/0.2-dev-removing-old-thrift eb77851ae -> 568721935


All unit test for core projects now pass.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/56872193
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/56872193
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/56872193

Branch: refs/heads/0.2-dev-removing-old-thrift
Commit: 568721935b56588bbfc775360f6a7d3e6dcb267b
Parents: ed67114
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sat Dec 1 23:16:50 2012 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sat Dec 1 23:16:50 2012 -0500

----------------------------------------------------------------------
 .../java/org/apache/blur/manager/IndexManager.java |  271 ---------------
 .../java/org/apache/blur/thrift/BlurServer.java    |    7 -
 .../org/apache/blur/thrift/ThriftBlurServer.java   |   14 +-
 .../org/apache/blur/manager/IndexManagerTest.java  |  101 ------
 .../apache/blur/search/RandomSuperQueryTest.java   |   42 +--
 .../org/apache/blur/search/SuperQueryTest.java     |   79 +++--
 src/pom.xml                                        |    8 +-
 7 files changed, 70 insertions(+), 452 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
deleted file mode 100644
index f83f596..0000000
--- a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ /dev/null
@@ -1,271 +0,0 @@
-package org.apache.blur.manager;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static org.apache.blur.utils.BlurConstants.ROW_ID;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TreeSet;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.blur.concurrent.Executors;
-import org.apache.blur.log.Log;
-import org.apache.blur.log.LogFactory;
-import org.apache.blur.manager.writer.BlurIndex;
-import org.apache.blur.metrics.BlurMetrics;
-import org.apache.blur.metrics.QueryMetrics;
-import org.apache.blur.thrift.BException;
-import org.apache.blur.thrift.generated.BlurException;
-import org.apache.blur.utils.BlurExecutorCompletionService;
-import org.apache.blur.utils.BlurUtil;
-import org.apache.blur.utils.ForkJoin;
-import org.apache.blur.utils.ForkJoin.Merger;
-import org.apache.blur.utils.ForkJoin.ParallelCall;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.StoredFieldVisitor;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.util.BytesRef;
-
-public class IndexManager {
-
-  private static final String NOT_FOUND = "NOT_FOUND";
-  private static final Log LOG = LogFactory.getLog(IndexManager.class);
-
-  private IndexServer _indexServer;
-  private ExecutorService _executor;
-  private ExecutorService _mutateExecutor;
-  private int _threadCount;
-  private boolean _closed;
-  private BlurPartitioner<BytesWritable, Void> _blurPartitioner = new BlurPartitioner<BytesWritable,
Void>();
-  private BlurFilterCache _filterCache = new DefaultBlurFilterCache();
-  private BlurMetrics _blurMetrics;
-  private QueryMetrics _queryMetrics;
-  private long _defaultParallelCallTimeout = TimeUnit.MINUTES.toMillis(1);
-
-  public void setMaxClauseCount(int maxClauseCount) {
-    BooleanQuery.setMaxClauseCount(maxClauseCount);
-  }
-
-  public void init() {
-    _executor = Executors.newThreadPool("index-manager", _threadCount);
-    // @TODO give the mutate it's own thread pool
-    _mutateExecutor = Executors.newThreadPool("index-manager-mutate", _threadCount);
-    _queryMetrics = QueryMetrics.getInstance();
-    LOG.info("Init Complete");
-  }
-
-  public synchronized void close() {
-    if (!_closed) {
-      _closed = true;
-      _executor.shutdownNow();
-      _mutateExecutor.shutdownNow();
-      _indexServer.close();
-    }
-  }
-
-  /**
-   * Location id format is <shard>/luceneid.
-   * 
-   * @param locationId
-   * @return
-   */
-  private String getShard(String locationId) {
-    String[] split = locationId.split("\\/");
-    if (split.length != 2) {
-      throw new IllegalArgumentException("Location id invalid [" + locationId + "]");
-    }
-    return split[0];
-  }
-
-  private static String getRowId(IndexReader reader, int docId) throws CorruptIndexException,
IOException {
-    reader.document(docId, new StoredFieldVisitor() {
-      @Override
-      public Status needsField(FieldInfo fieldInfo) throws IOException {
-        if (ROW_ID.equals(fieldInfo.name)) {
-          return StoredFieldVisitor.Status.STOP;
-        }
-        return StoredFieldVisitor.Status.NO;
-      }
-    });
-    return reader.document(docId).get(ROW_ID);
-  }
-
-  private static String getColumnName(String fieldName) {
-    return fieldName.substring(fieldName.lastIndexOf('.') + 1);
-  }
-
-  private static String getColumnFamily(String fieldName) {
-    return fieldName.substring(0, fieldName.lastIndexOf('.'));
-  }
-
-  public IndexServer getIndexServer() {
-    return _indexServer;
-  }
-
-  public void setIndexServer(IndexServer indexServer) {
-    this._indexServer = indexServer;
-  }
-
-  public long recordFrequency(final String table, final String columnFamily, final String
columnName, final String value) throws Exception {
-    Map<String, BlurIndex> blurIndexes;
-    try {
-      blurIndexes = _indexServer.getIndexes(table);
-    } catch (IOException e) {
-      LOG.error("Unknown error while trying to fetch index readers.", e);
-      throw new BException(e.getMessage(), e);
-    }
-    return ForkJoin.execute(_executor, blurIndexes.entrySet(), new ParallelCall<Entry<String,
BlurIndex>, Long>() {
-      @Override
-      public Long call(Entry<String, BlurIndex> input) throws Exception {
-        BlurIndex index = input.getValue();
-        IndexReader reader = index.getIndexReader();
-        try {
-          return recordFrequency(reader, columnFamily, columnName, value);
-        } finally {
-          // this will allow for closing of index
-          reader.decRef();
-        }
-      }
-    }).merge(new Merger<Long>() {
-      @Override
-      public Long merge(BlurExecutorCompletionService<Long> service) throws BlurException
{
-        long total = 0;
-        while (service.getRemainingCount() > 0) {
-          Future<Long> future = service.poll(_defaultParallelCallTimeout, TimeUnit.MILLISECONDS,
true, table, columnFamily, columnName, value);
-          total += service.getResultThrowException(future, table, columnFamily, columnName,
value);
-        }
-        return total;
-      }
-    });
-  }
-
-  public List<String> terms(final String table, final String columnFamily, final String
columnName, final String startWith, final short size) throws Exception {
-    Map<String, BlurIndex> blurIndexes;
-    try {
-      blurIndexes = _indexServer.getIndexes(table);
-    } catch (IOException e) {
-      LOG.error("Unknown error while trying to fetch index readers.", e);
-      throw new BException(e.getMessage(), e);
-    }
-    return ForkJoin.execute(_executor, blurIndexes.entrySet(), new ParallelCall<Entry<String,
BlurIndex>, List<String>>() {
-      @Override
-      public List<String> call(Entry<String, BlurIndex> input) throws Exception
{
-        BlurIndex index = input.getValue();
-        IndexReader reader = index.getIndexReader();
-        try {
-          return terms(reader, columnFamily, columnName, startWith, size);
-        } finally {
-          // this will allow for closing of index
-          reader.decRef();
-        }
-      }
-    }).merge(new Merger<List<String>>() {
-      @Override
-      public List<String> merge(BlurExecutorCompletionService<List<String>>
service) throws BlurException {
-        TreeSet<String> terms = new TreeSet<String>();
-        while (service.getRemainingCount() > 0) {
-          Future<List<String>> future = service.poll(_defaultParallelCallTimeout,
TimeUnit.MILLISECONDS, true, table, columnFamily, columnName, startWith, size);
-          terms.addAll(service.getResultThrowException(future, table, columnFamily, columnName,
startWith, size));
-        }
-        return new ArrayList<String>(terms).subList(0, Math.min(size, terms.size()));
-      }
-    });
-  }
-
-  public static long recordFrequency(IndexReader reader, String columnFamily, String columnName,
String value) throws IOException {
-    return reader.docFreq(getTerm(columnFamily, columnName, value));
-  }
-
-  public static List<String> terms(IndexReader reader, String columnFamily, String
columnName, String startWith, short size) throws IOException {
-    Term term = getTerm(columnFamily, columnName, startWith);
-    List<String> terms = new ArrayList<String>(size);
-    AtomicReader areader = BlurUtil.getAtomicReader(reader);
-    Terms termsAll = areader.terms(term.field());
-    TermsEnum termEnum = termsAll.iterator(null);
-    BytesRef currentTermText;
-    while ((currentTermText = termEnum.next()) != null) {
-      terms.add(currentTermText.utf8ToString());
-      if (terms.size() >= size) {
-        return terms;
-      }
-    }
-    return terms;
-  }
-
-  private static Term getTerm(String columnFamily, String columnName, String value) {
-    if (columnName == null) {
-      throw new NullPointerException("ColumnName cannot both be null.");
-    }
-    if (columnFamily == null) {
-      return new Term(columnName, value);
-    }
-    return new Term(columnFamily + "." + columnName, value);
-  }
-
-  public void setStatusCleanupTimerDelay(long delay) {
-  }
-
-  private int getNumberOfShards(String table) {
-    return _indexServer.getShardCount(table);
-  }
-
-  public void setThreadCount(int threadCount) {
-    this._threadCount = threadCount;
-  }
-
-  public void setBlurMetrics(BlurMetrics blurMetrics) {
-    _blurMetrics = blurMetrics;
-  }
-
-  public void setFilterCache(BlurFilterCache filterCache) {
-    _filterCache = filterCache;
-  }
-
-  public void optimize(String table, int numberOfSegmentsPerShard) throws BException {
-    Map<String, BlurIndex> blurIndexes;
-    try {
-      blurIndexes = _indexServer.getIndexes(table);
-    } catch (IOException e) {
-      LOG.error("Unknown error while trying to fetch index readers.", e);
-      throw new BException(e.getMessage(), e);
-    }
-
-    Collection<BlurIndex> values = blurIndexes.values();
-    for (BlurIndex index : values) {
-      try {
-        index.optimize(numberOfSegmentsPerShard);
-      } catch (IOException e) {
-        LOG.error("Unknown error while trying to optimize indexes.", e);
-        throw new BException(e.getMessage(), e);
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java
index 3bf8d2a..0d9c452 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java
@@ -44,7 +44,6 @@ import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.concurrent.Executors;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
-import org.apache.blur.manager.IndexManager;
 import org.apache.blur.manager.IndexServer;
 import org.apache.blur.manager.writer.BlurIndex;
 import org.apache.blur.thrift.BlurServer.SearchAction.ACTION;
@@ -79,7 +78,6 @@ import org.apache.thrift.TException;
 public class BlurServer extends TableAdmin implements Iface {
 
   private static final Log LOG = LogFactory.getLog(BlurServer.class);
-  private IndexManager _indexManager;
   private IndexServer _indexServer;
   private boolean _closed;
   private long _maxTimeToLive = TimeUnit.MINUTES.toMillis(1);
@@ -123,7 +121,6 @@ public class BlurServer extends TableAdmin implements Iface {
   public synchronized void close() {
     if (!_closed) {
       _closed = true;
-      _indexManager.close();
       _dataFetch.shutdownNow();
     }
   }
@@ -144,10 +141,6 @@ public class BlurServer extends TableAdmin implements Iface {
     _maxQueryCacheElements = maxQueryCacheElements;
   }
 
-  public void setIndexManager(IndexManager indexManager) {
-    _indexManager = indexManager;
-  }
-
   public void setIndexServer(IndexServer indexServer) {
     _indexServer = indexServer;
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java
index 23d3d46..064e99f 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurServer.java
@@ -21,8 +21,6 @@ import static org.apache.blur.utils.BlurConstants.BLUR_CLUSTER_NAME;
 import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_BIND_PORT;
 import static org.apache.blur.utils.BlurConstants.BLUR_GUI_CONTROLLER_PORT;
 import static org.apache.blur.utils.BlurConstants.BLUR_GUI_SHARD_PORT;
-import static org.apache.blur.utils.BlurConstants.BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT;
-import static org.apache.blur.utils.BlurConstants.BLUR_MAX_CLAUSE_COUNT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_ADDRESS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION;
@@ -52,7 +50,6 @@ import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.manager.BlurFilterCache;
 import org.apache.blur.manager.DefaultBlurFilterCache;
-import org.apache.blur.manager.IndexManager;
 import org.apache.blur.manager.clusterstatus.ZookeeperClusterStatus;
 import org.apache.blur.manager.indexserver.BlurIndexWarmup;
 import org.apache.blur.manager.indexserver.BlurServerShutDown;
@@ -195,14 +192,6 @@ public class ThriftBlurServer extends ThriftServer {
     indexServer.setTimeBetweenRefreshs(configuration.getLong(BLUR_SHARD_TIME_BETWEEN_REFRESHS,
500));
     indexServer.init();
 
-    final IndexManager indexManager = new IndexManager();
-    indexManager.setIndexServer(indexServer);
-    indexManager.setMaxClauseCount(configuration.getInt(BLUR_MAX_CLAUSE_COUNT, 1024));
-    indexManager.setThreadCount(configuration.getInt(BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT,
32));
-    indexManager.setBlurMetrics(blurMetrics);
-    indexManager.setFilterCache(filterCache);
-    indexManager.init();
-
     TableLayout layout = new TableLayout() {
       @Override
       public String findServer(String table, int shard, TYPE type) {
@@ -230,7 +219,6 @@ public class ThriftBlurServer extends ThriftServer {
 
     final BlurServer shardServer = new BlurServer();
     shardServer.setIndexServer(indexServer);
-    shardServer.setIndexManager(indexManager);
     shardServer.setZookeeper(zooKeeper);
     shardServer.setClusterStatus(clusterStatus);
     shardServer.setConfiguration(configuration);
@@ -256,7 +244,7 @@ public class ThriftBlurServer extends ThriftServer {
       @Override
       public void shutdown() {
         ThreadWatcher threadWatcher = ThreadWatcher.instance();
-        quietClose(refresher, server, shardServer, indexManager, indexServer, threadWatcher,
clusterStatus, zooKeeper, httpServer);
+        quietClose(refresher, server, shardServer, indexServer, threadWatcher, clusterStatus,
zooKeeper, httpServer);
       }
     };
     server.setShutdown(shutdown);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
deleted file mode 100644
index 82e3d92..0000000
--- a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-package org.apache.blur.manager;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.blur.manager.indexserver.LocalIndexServer;
-import org.apache.blur.metrics.BlurMetrics;
-import org.apache.blur.thrift.generated.BlurException;
-import org.apache.blur.utils.BlurConstants;
-import org.apache.blur.utils.BlurUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-
-public class IndexManagerTest {
-
-  private static final String SHARD_NAME = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX,
0);
-  private static final String TABLE = "table";
-  private static final String FAMILY = "test-family";
-  private static final String FAMILY2 = "test-family2";
-
-  private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp"));
-
-  private LocalIndexServer server;
-  private IndexManager indexManager;
-
-  @Before
-  public void setUp() throws BlurException, IOException, InterruptedException {
-    File file = new File(TMPDIR, "indexer-manager-test");
-    rm(file);
-    new File(new File(file, TABLE), SHARD_NAME).mkdirs();
-    server = new LocalIndexServer(file, new Path(file.getAbsolutePath()));
-
-    indexManager = new IndexManager();
-    indexManager.setStatusCleanupTimerDelay(1000);
-    indexManager.setIndexServer(server);
-    indexManager.setThreadCount(1);
-    indexManager.setBlurMetrics(new BlurMetrics(new Configuration()));
-    indexManager.init();
-    setupData();
-  }
-
-  @After
-  public void teardown() {
-    indexManager.close();
-    indexManager = null;
-    server = null;
-  }
-
-  private void rm(File file) {
-    if (file.isDirectory()) {
-      for (File f : file.listFiles()) {
-        rm(f);
-      }
-    }
-    file.delete();
-  }
-
-  private void setupData() throws BlurException, IOException {
-
-  }
-
-  @Test
-  public void testRecordFrequency() throws Exception {
-    assertEquals(2, indexManager.recordFrequency(TABLE, FAMILY, "testcol1", "value1"));
-    assertEquals(0, indexManager.recordFrequency(TABLE, FAMILY, "testcol1", "NO VALUE"));
-  }
-
-
-  @Test
-  public void testTerms() throws Exception {
-    List<String> terms = indexManager.terms(TABLE, FAMILY, "testcol1", "", (short)
100);
-    assertEquals(Arrays.asList("value1", "value13", "value16", "value4", "value7"), terms);
-  }
-
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
b/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
index 7dc9c71..8c3b8bb 100644
--- a/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
+++ b/src/blur-core/src/test/java/org/apache/blur/search/RandomSuperQueryTest.java
@@ -29,6 +29,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 
+import org.apache.blur.lucene.search.ScoreType;
+import org.apache.blur.lucene.search.SuperQuery;
+import org.apache.blur.utils.BlurConstants;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field.Store;
@@ -38,12 +41,13 @@ import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.ParseException;
-import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.QueryWrapperFilter;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.LockObtainFailedException;
@@ -60,7 +64,7 @@ public class RandomSuperQueryTest {
   private static final int MAX_NUM_COLS = 21;// 21
   private static final int MIN_NUM_COLS = 3;// 3
   private static final int MAX_NUM_OF_WORDS = 1000;
-  private static final int MOD_USED_FOR_SAMPLING = 1;//
+  private static final int MOD_USED_FOR_SAMPLING = 7;//
 
   private Random seedGen = new Random(1);
 
@@ -68,10 +72,8 @@ public class RandomSuperQueryTest {
   public void testRandomSuperQuery() throws CorruptIndexException, IOException, InterruptedException,
ParseException {
     long seed = seedGen.nextLong();
 
-    Filter filter = new QueryWrapperFilter(new MatchAllDocsQuery());
-
     Random random = new Random(seed);
-    Collection<String> sampler = new HashSet<String>();
+    Collection<Query> sampler = new HashSet<Query>();
     System.out.print("Creating index... ");
     System.out.flush();
     Directory directory = createIndex(random, sampler);
@@ -81,10 +83,7 @@ public class RandomSuperQueryTest {
     assertTrue(!sampler.isEmpty());
     IndexSearcher searcher = new IndexSearcher(reader);
     long s = System.currentTimeMillis();
-    for (String str : sampler) {
-      Query query = null;// new SuperParser(LUCENE_VERSION, new BlurAnalyzer(),
-                         // true, filter, ScoreType.AGGREGATE).parse(str);
-      
+    for (Query query : sampler) {
       TopDocs topDocs = searcher.search(query, 10);
       assertTrue("seed [" + seed + "] {" + query + "} {" + s + "}", topDocs.totalHits >
0);
     }
@@ -92,7 +91,7 @@ public class RandomSuperQueryTest {
     System.out.println("Finished in [" + (e - s) + "] ms");
   }
 
-  private Directory createIndex(Random random, Collection<String> sampler) throws CorruptIndexException,
LockObtainFailedException, IOException {
+  private Directory createIndex(Random random, Collection<Query> sampler) throws CorruptIndexException,
LockObtainFailedException, IOException {
     Directory directory = new RAMDirectory();
     String[] columnFamilies = genWords(random, MIN_NUM_COL_FAM, MAX_NUM_COL_FAM, "colfam");
     Map<String, String[]> columns = new HashMap<String, String[]>();
@@ -117,33 +116,32 @@ public class RandomSuperQueryTest {
     return str;
   }
 
-  private List<Document> generatSuperDoc(Random random, Map<String, String[]>
columns, Collection<String> sampler) {
+  private List<Document> generatSuperDoc(Random random, Map<String, String[]>
columns, Collection<Query> sampler) {
     List<Document> docs = new ArrayList<Document>();
-    StringBuilder builder = new StringBuilder();
+    BooleanQuery booleanQuery = new BooleanQuery();
     for (String colFam : columns.keySet()) {
       String[] cols = columns.get(colFam);
       for (int i = 0; i < random.nextInt(MAX_NUM_DOCS_PER_COL_FAM); i++) {
         Document doc = new Document();
-        int staringLength = builder.length();
         for (String column : cols) {
           if (random.nextInt() % MOD_COLS_USED_FOR_SKIPPING == 0) {
             String word = genWord(random, "word");
             doc.add(new StringField(colFam + "." + column, word, Store.YES));
             if (random.nextInt() % MOD_USED_FOR_SAMPLING == 0) {
-              builder.append(" +" + colFam + "." + column + ":" + word);
+              TermQuery termQuery = new TermQuery(new Term(colFam + "." + column, word));
+              SuperQuery query = new SuperQuery(termQuery, ScoreType.SUPER);
+              booleanQuery.add(query, Occur.MUST);
             }
           }
         }
-        if (builder.length() != staringLength) {
-          builder.append(" nojoin.nojoin ");
-        }
         docs.add(doc);
       }
     }
-    String string = builder.toString().trim();
-    if (!string.isEmpty()) {
-      sampler.add(string);
+    if (!booleanQuery.clauses().isEmpty()) {
+      sampler.add(booleanQuery);
     }
+    Document document = docs.get(0);
+    document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE,
Store.NO));
     return docs;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java b/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java
index a95cb82..52dae72 100644
--- a/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java
+++ b/src/blur-core/src/test/java/org/apache/blur/search/SuperQueryTest.java
@@ -18,17 +18,28 @@ package org.apache.blur.search;
  */
 
 import static junit.framework.Assert.assertEquals;
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
 import static org.apache.blur.utils.BlurConstants.ROW_ID;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicLongArray;
 
 import org.apache.blur.lucene.search.FacetQuery;
 import org.apache.blur.lucene.search.ScoreType;
 import org.apache.blur.lucene.search.SuperQuery;
+import org.apache.blur.utils.BlurConstants;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
@@ -41,17 +52,11 @@ import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.store.RAMDirectory;
 import org.junit.Test;
 
-
 public class SuperQueryTest {
 
   private static final String PERSON_NAME = "person.name";
   private static final String ADDRESS_STREET = "address.street";
 
-  private static final String STREET = "street";
-  private static final String ADDRESS = "address";
-  private static final String PERSON = "person";
-  private static final String NAME = "name";
-
   private static final String NAME1 = "jon";
   private static final String NAME2 = "jane";
   private static final String STREET2 = "main st";
@@ -82,9 +87,6 @@ public class SuperQueryTest {
     TopDocs topDocs = searcher.search(booleanQuery, 10);
     printTopDocs(topDocs);
     assertEquals(3, topDocs.totalHits);
-    assertEquals(3.30, topDocs.scoreDocs[0].score, 0.01);
-    assertEquals(2.20, topDocs.scoreDocs[1].score, 0.01);
-    assertEquals(0.55, topDocs.scoreDocs[2].score, 0.01);
   }
 
   @Test
@@ -96,9 +98,6 @@ public class SuperQueryTest {
     TopDocs topDocs = searcher.search(booleanQuery, 10);
     assertEquals(3, topDocs.totalHits);
     printTopDocs(topDocs);
-    assertEquals(2.20, topDocs.scoreDocs[0].score, 0.01);
-    assertEquals(2.20, topDocs.scoreDocs[1].score, 0.01);
-    assertEquals(0.55, topDocs.scoreDocs[2].score, 0.01);
   }
 
   private void printTopDocs(TopDocs topDocs) {
@@ -117,9 +116,6 @@ public class SuperQueryTest {
     TopDocs topDocs = searcher.search(booleanQuery, 10);
     assertEquals(3, topDocs.totalHits);
     printTopDocs(topDocs);
-    assertEquals(2.0, topDocs.scoreDocs[0].score, 0.01);
-    assertEquals(2.0, topDocs.scoreDocs[1].score, 0.01);
-    assertEquals(0.5, topDocs.scoreDocs[2].score, 0.01);
   }
 
   @Test
@@ -131,9 +127,6 @@ public class SuperQueryTest {
     TopDocs topDocs = searcher.search(booleanQuery, 10);
     assertEquals(3, topDocs.totalHits);
     printTopDocs(topDocs);
-    assertEquals(3.10, topDocs.scoreDocs[0].score, 0.01);
-    assertEquals(3.00, topDocs.scoreDocs[1].score, 0.01);
-    assertEquals(0.75, topDocs.scoreDocs[2].score, 0.01);
   }
 
   @Test
@@ -154,9 +147,6 @@ public class SuperQueryTest {
     TopDocs topDocs = searcher.search(query, 10);
     assertEquals(3, topDocs.totalHits);
     printTopDocs(topDocs);
-    assertEquals(3.10, topDocs.scoreDocs[0].score, 0.01);
-    assertEquals(3.00, topDocs.scoreDocs[1].score, 0.01);
-    assertEquals(0.75, topDocs.scoreDocs[2].score, 0.01);
   }
 
   private static IndexSearcher createSearcher() throws Exception {
@@ -167,22 +157,43 @@ public class SuperQueryTest {
 
   public static Directory createIndex() throws CorruptIndexException, LockObtainFailedException,
IOException {
     Directory directory = new RAMDirectory();
-//    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION,
new StandardAnalyzer(LUCENE_VERSION)));
-//    BlurAnalyzer analyzer = new BlurAnalyzer(new StandardAnalyzer(LUCENE_VERSION));
-//    RowIndexWriter indexWriter = new RowIndexWriter(writer, analyzer);
-//    indexWriter.replace(
-//        false,
-//        newRow("1", newRecord(PERSON, UUID.randomUUID().toString(), newColumn(NAME, NAME1)),
newRecord(PERSON, UUID.randomUUID().toString(), newColumn(NAME, NAME1)),
-//            newRecord(ADDRESS, UUID.randomUUID().toString(), newColumn(STREET, STREET1))));
-//    indexWriter.replace(false,
-//        newRow("2", newRecord(PERSON, UUID.randomUUID().toString(), newColumn(NAME, NAME2)),
newRecord(ADDRESS, UUID.randomUUID().toString(), newColumn(STREET, STREET1))));
-//    indexWriter.replace(false,
-//        newRow("3", newRecord(PERSON, UUID.randomUUID().toString(), newColumn(NAME, NAME1)),
newRecord(ADDRESS, UUID.randomUUID().toString(), newColumn(STREET, STREET2))));
-//    ;
-//    writer.close();
+    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION,
new StandardAnalyzer(LUCENE_VERSION)));
+    writer.addDocuments(
+        addPrime(Arrays.asList(
+            newDocument(newStringField(ROW_ID, "1"), newStringField(PERSON_NAME, NAME1)),

+            newDocument(newStringField(ROW_ID, "1"), newStringField(PERSON_NAME, NAME1)),
+            newDocument(newStringField(ROW_ID, "1"), newStringField(ADDRESS_STREET, STREET1)))));
+    writer.addDocuments(
+        addPrime(Arrays.asList(
+            newDocument(newStringField(ROW_ID, "2"), newStringField(PERSON_NAME, NAME2)),

+            newDocument(newStringField(ROW_ID, "2"), newStringField(ADDRESS_STREET, STREET1)))));
+    writer.addDocuments(
+        addPrime(Arrays.asList(
+            newDocument(newStringField(ROW_ID, "3"), newStringField(PERSON_NAME, NAME1)),

+            newDocument(newStringField(ROW_ID, "3"), newStringField(ADDRESS_STREET, STREET1)),

+            newDocument(newStringField(ROW_ID, "3"), newStringField(ADDRESS_STREET, STREET2)))));
+    writer.close();
     return directory;
   }
 
+  private static List<Document> addPrime(List<Document> docs) {
+    Document document = docs.get(0);
+    document.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE,
Store.NO));
+    return docs;
+  }
+
+  private static Document newDocument(IndexableField... fields) {
+    Document document = new Document();
+    for (IndexableField field : fields) {
+      document.add(field);
+    }
+    return document;
+  }
+
+  private static IndexableField newStringField(String name, String value) {
+    return new StringField(name, value, Store.YES);
+  }
+
   private Query wrapSuper(Query query) {
     return new SuperQuery(query, ScoreType.AGGREGATE);
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/56872193/src/pom.xml
----------------------------------------------------------------------
diff --git a/src/pom.xml b/src/pom.xml
index a40804b..fc67591 100644
--- a/src/pom.xml
+++ b/src/pom.xml
@@ -41,12 +41,12 @@ under the License.
 		<module>blur-core</module>
 		<module>blur-thrift</module>
 		<module>blur-store</module>
-		<module>blur-mapred</module>
+		<!--module>blur-mapred</module-->
 		<module>blur-util</module>
 		<module>blur-gui</module>
-		<module>blur-jdbc</module>
-		<module>blur-testsuite</module>
-		<module>blur-shell</module>
+		<!--module>blur-jdbc</module-->
+		<!--module>blur-testsuite</module-->
+		<!--module>blur-shell</module-->
 	</modules>
 
 	<dependencies>


Mime
View raw message