incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [5/6] git commit: Updates to try and get unit tests to pass. IndexManagerTest still has 3 broken tests.
Date Thu, 11 Apr 2013 03:10:53 GMT
Updates to try and get unit tests to pass.  IndexManagerTest still has 3 broken tests.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/bc1e033a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/bc1e033a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/bc1e033a

Branch: refs/heads/0.1.5
Commit: bc1e033a9b1e6b22ef24719ec14d17e72f8cdfff
Parents: c1231e3
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed Apr 10 23:09:31 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed Apr 10 23:09:31 2013 -0400

----------------------------------------------------------------------
 .../java/org/apache/blur/manager/IndexManager.java |   20 ++-
 .../apache/blur/thrift/ThriftBlurShardServer.java  |    3 +-
 .../java/org/apache/blur/thrift/ThriftServer.java  |    4 +
 .../src/test/java/org/apache/blur/MiniCluster.java |   35 +++--
 .../org/apache/blur/manager/IndexManagerTest.java  |  102 ++++++++++++++-
 .../blur/lucene/search/RandomSuperQueryTest.java   |    2 +-
 .../apache/blur/lucene/search/SuperQueryTest.java  |    2 +-
 .../blur/lucene/search/TestingPagingCollector.java |    2 +-
 8 files changed, 143 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 0e82f57..2d68f99 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -342,8 +342,8 @@ public class IndexManager {
       TableContext context = getTableContext(table);
       if (isSimpleQuery(blurQuery)) {
         SimpleQuery simpleQuery = blurQuery.simpleQuery;
-        Filter preFilter = QueryParserUtil
-            .parseFilter(table, simpleQuery.preSuperFilter, false, analyzer, _filterCache,
context);
+        Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.preSuperFilter,
false, analyzer,
+            _filterCache, context);
         Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.postSuperFilter,
true, analyzer,
             _filterCache, context);
         Query userQuery = QueryParserUtil.parseQuery(simpleQuery.queryStr, simpleQuery.superQueryOn,
analyzer,
@@ -395,15 +395,16 @@ public class IndexManager {
     return false;
   }
 
-  private Query getFacetedQuery(BlurQuery blurQuery, Query userQuery, AtomicLongArray counts,
BlurAnalyzer analyzer, TableContext context)
-      throws ParseException {
+  private Query getFacetedQuery(BlurQuery blurQuery, Query userQuery, AtomicLongArray counts,
BlurAnalyzer analyzer,
+      TableContext context) throws ParseException {
     if (blurQuery.facets == null) {
       return userQuery;
     }
     return new FacetQuery(userQuery, getFacetQueries(blurQuery, analyzer, context), counts);
   }
 
-  private Query[] getFacetQueries(BlurQuery blurQuery, BlurAnalyzer analyzer, TableContext
context) throws ParseException {
+  private Query[] getFacetQueries(BlurQuery blurQuery, BlurAnalyzer analyzer, TableContext
context)
+      throws ParseException {
     int size = blurQuery.facets.size();
     Query[] queries = new Query[size];
     for (int i = 0; i < size; i++) {
@@ -985,8 +986,8 @@ public class IndexManager {
         // IndexSearcher searcher = new IndexSearcher(escapeReader);
         searcher.setSimilarity(_indexServer.getSimilarity(_table));
         Query rewrite = searcher.rewrite((Query) _query.clone());
-        
-        //BlurResultIterableSearcher will close searcher.
+
+        // BlurResultIterableSearcher will close searcher.
         return new BlurResultIterableSearcher(_running, rewrite, _table, shard, searcher,
_selector);
       } finally {
         _queriesInternalMeter.mark();
@@ -1022,4 +1023,9 @@ public class IndexManager {
       }
     }
   }
+
+  public void setClusterStatus(ClusterStatus clusterStatus) {
+    _clusterStatus = clusterStatus;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index ab958bf..1beb382 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -100,8 +100,6 @@ public class ThriftBlurShardServer extends ThriftServer {
     int bindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1);
     bindPort += serverIndex;
     
-//    BlurMetrics blurMetrics = new BlurMetrics(config);
-    
     int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_SHARD_PORT));
     final HttpJettyServer httpServer;
     if (baseGuiPort > 0) {
@@ -192,6 +190,7 @@ public class ThriftBlurShardServer extends ThriftServer {
     indexManager.setMaxClauseCount(configuration.getInt(BLUR_MAX_CLAUSE_COUNT, 1024));
     indexManager.setThreadCount(configuration.getInt(BLUR_INDEXMANAGER_SEARCH_THREAD_COUNT,
32));
     indexManager.setFilterCache(filterCache);
+    indexManager.setClusterStatus(clusterStatus);
     indexManager.init();
 
     final BlurShardServer shardServer = new BlurShardServer();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
index 787b034..1beba2b 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java
@@ -26,6 +26,7 @@ import java.net.UnknownHostException;
 import java.util.concurrent.ExecutorService;
 
 import org.apache.blur.BlurConfiguration;
+import org.apache.blur.concurrent.Executors;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.manager.indexserver.BlurServerShutDown.BlurShutdown;
@@ -103,13 +104,16 @@ public class ThriftServer {
   }
 
   public void start() throws TTransportException {
+    _executorService = Executors.newThreadPool("thrift-processors", _threadCount);
     Blur.Processor<Blur.Iface> processor = new Blur.Processor<Blur.Iface>(_iface);
      TServerSocket serverTransport = new
      TServerSocket(getBindInetSocketAddress(_configuration));
     
      TThreadPoolServer.Args args = new
      TThreadPoolServer.Args(serverTransport);
+     
      args.processor(processor);
+     args.executorService(_executorService);
      args.transportFactory(new TFramedTransport.Factory());
      args.protocolFactory(new TBinaryProtocol.Factory(true, true));
      _server = new TThreadPoolServer(args);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java
index 1da7641..94ed435 100644
--- a/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java
+++ b/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java
@@ -75,7 +75,6 @@ import org.apache.zookeeper.server.ZooKeeperServerMain;
 import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
 import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException;
 
-
 public abstract class MiniCluster {
 
   private static Log LOG = LogFactory.getLog(MiniCluster.class);
@@ -87,19 +86,20 @@ public abstract class MiniCluster {
   private static List<ThriftServer> shards = new ArrayList<ThriftServer>();
   private static String controllerConnectionStr;
 
-  public static void main(String[] args) throws IOException, InterruptedException, KeeperException,
BlurException, TException {
+  public static void main(String[] args) throws IOException, InterruptedException, KeeperException,
BlurException,
+      TException {
     startDfs("./tmp");
     startZooKeeper("./tmp");
     startControllers(1);
     startShards(1);
-    
-    //Run the controllers/shards on custom ports.  
-    //BlurConfiguration conf = new BlurConfiguration(false);
-    //conf.setInt(BLUR_CONTROLLER_BIND_PORT, 40001);
-    //conf.setInt(BLUR_SHARD_BIND_PORT, 40002);
-    //startControllers(conf, 1);
-    //startShards(conf, 1);
-    
+
+    // Run the controllers/shards on custom ports.
+    // BlurConfiguration conf = new BlurConfiguration(false);
+    // conf.setInt(BLUR_CONTROLLER_BIND_PORT, 40001);
+    // conf.setInt(BLUR_SHARD_BIND_PORT, 40002);
+    // startControllers(conf, 1);
+    // startShards(conf, 1);
+
     try {
       Iface client = BlurClient.getClient(getControllerConnectionStr());
       createTable("test", client);
@@ -201,13 +201,13 @@ public abstract class MiniCluster {
 
   private static BlurConfiguration getBlurConfiguration(BlurConfiguration overrides) {
     BlurConfiguration conf = getBlurConfiguration();
-	
-    for(Map.Entry<String, String> over: overrides.getProperties().entrySet()) {
+
+    for (Map.Entry<String, String> over : overrides.getProperties().entrySet()) {
       conf.set(over.getKey().toString(), over.getValue().toString());
     }
     return conf;
   }
-  
+
   private static BlurConfiguration getBlurConfiguration() {
     BlurConfiguration configuration;
     try {
@@ -457,7 +457,8 @@ public abstract class MiniCluster {
       for (int i = 0; i < enumerate; i++) {
         Thread thread = threads[i];
         if (thread.getName().startsWith("pool")) {
-          while (thread.isAlive()) {
+          if (thread.isAlive()) {
+            thread.interrupt();
             LOG.info("Stopping ThreadPoolExecutor [" + thread.getName() + "]");
             Object target = getField(thread, "target");
             if (target != null) {
@@ -466,6 +467,12 @@ public abstract class MiniCluster {
                 e.shutdownNow();
               }
             }
+            try {
+              LOG.info("Waiting for thread pool to exit [" + thread.getName() + "]");
+              thread.join();
+            } catch (InterruptedException e) {
+              throw new RuntimeException(e);
+            }
           }
         }
       }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
index 0056e36..ebd8303 100644
--- a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
+++ b/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
@@ -45,6 +45,7 @@ import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicLongArray;
 
+import org.apache.blur.manager.clusterstatus.ClusterStatus;
 import org.apache.blur.manager.indexserver.LocalIndexServer;
 import org.apache.blur.manager.results.BlurResultIterable;
 import org.apache.blur.thrift.generated.AnalyzerDefinition;
@@ -92,7 +93,7 @@ public class IndexManagerTest {
     File file = new File(base, TABLE);
     file.mkdirs();
 
-    TableDescriptor tableDescriptor = new TableDescriptor();
+    final TableDescriptor tableDescriptor = new TableDescriptor();
     tableDescriptor.setName(TABLE);
     tableDescriptor.setTableUri(file.toURI().toString());
     tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
@@ -104,6 +105,103 @@ public class IndexManagerTest {
     indexManager.setStatusCleanupTimerDelay(1000);
     indexManager.setIndexServer(server);
     indexManager.setThreadCount(1);
+    indexManager.setClusterStatus(new ClusterStatus() {
+      
+      @Override
+      public void removeTable(String cluster, String table, boolean deleteIndexFiles) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public boolean isReadOnly(boolean useCache, String cluster, String table) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public boolean isOpen() {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public boolean isInSafeMode(boolean useCache, String cluster) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public boolean isEnabled(boolean useCache, String cluster, String table) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public boolean isBlockCacheEnabled(String cluster, String table) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public List<String> getTableList(boolean useCache, String cluster) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public TableDescriptor getTableDescriptor(boolean useCache, String cluster, String
table) {
+        return tableDescriptor;
+      }
+      
+      @Override
+      public List<String> getShardServerList(String cluster) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public int getShardCount(boolean useCache, String cluster, String table) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public List<String> getOnlineShardServers(boolean useCache, String cluster) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public List<String> getControllerServerList() {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public List<String> getClusterList(boolean useCache) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public String getCluster(boolean useCache, String table) {
+        return BlurConstants.BLUR_CLUSTER;
+      }
+      
+      @Override
+      public Set<String> getBlockCacheFileTypes(String cluster, String table) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public boolean exists(boolean useCache, String cluster, String table) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public void enableTable(String cluster, String table) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public void disableTable(String cluster, String table) {
+        throw new RuntimeException("Not impl");
+      }
+      
+      @Override
+      public void createTable(TableDescriptor tableDescriptor) {
+        throw new RuntimeException("Not impl");
+      }
+    });
     indexManager.init();
     setupData();
   }
@@ -161,6 +259,8 @@ public class IndexManagerTest {
     BlurQuery blurQuery = new BlurQuery();
     blurQuery.simpleQuery = new SimpleQuery();
     blurQuery.simpleQuery.queryStr = "+test-family.testcol12:value101 +test-family.testcol13:value102
+test-family2.testcol18:value501";
+//    blurQuery.simpleQuery.queryStr = "+super:<+test-family.testcol12:value101 +test-family.testcol13:value102>
+test-family2.testcol18:value501";
+    
     blurQuery.simpleQuery.superQueryOn = true;
     blurQuery.simpleQuery.type = ScoreType.SUPER;
     blurQuery.fetch = 10;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java
b/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java
index 707916c..ceaca80 100644
--- a/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java
+++ b/src/blur-query/src/test/java/org/apache/blur/lucene/search/RandomSuperQueryTest.java
@@ -1,4 +1,4 @@
-package org.apache.blur.search;
+package org.apache.blur.lucene.search;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperQueryTest.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperQueryTest.java
b/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperQueryTest.java
index fffb348..37346c9 100644
--- a/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperQueryTest.java
+++ b/src/blur-query/src/test/java/org/apache/blur/lucene/search/SuperQueryTest.java
@@ -1,4 +1,4 @@
-package org.apache.blur.search;
+package org.apache.blur.lucene.search;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/bc1e033a/src/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java
----------------------------------------------------------------------
diff --git a/src/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java
b/src/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java
index 5eaad3f..9358eb2 100644
--- a/src/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java
+++ b/src/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java
@@ -1,4 +1,4 @@
-package org.apache.blur.search;
+package org.apache.blur.lucene.search;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more


Mime
View raw message