incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/3] git commit: Fixing more unit tests.
Date Fri, 07 Jun 2013 14:30:19 GMT
Updated Branches:
  refs/heads/0.1.5 ed04d5bbc -> ae49a010a


Fixing more unit tests.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/b73a6ace
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/b73a6ace
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/b73a6ace

Branch: refs/heads/0.1.5
Commit: b73a6ace343e3e151570955dc64f1b8c39ed8531
Parents: ed04d5b
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Jun 6 22:45:25 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Jun 6 22:45:25 2013 -0400

----------------------------------------------------------------------
 .../blur/manager/writer/BlurIndexReader.java       |   36 ++++----
 .../apache/blur/manager/writer/BlurNRTIndex.java   |    3 +-
 .../apache/blur/server/IndexSearcherClosable.java  |   36 ++++----
 .../blur/server/IndexSearcherClosableNRT.java      |   49 ++++++++++
 .../blur/manager/indexserver/SafeModeTest.java     |    5 +-
 .../blur/manager/writer/BlurIndexReaderTest.java   |   71 ++++++++-------
 6 files changed, 130 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b73a6ace/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
index 431a829..f31fa38 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexReader.java
@@ -31,10 +31,7 @@ import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
 import org.apache.blur.thrift.generated.Row;
 import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.TieredMergePolicy;
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.store.Directory;
 
 public class BlurIndexReader extends BlurIndex {
@@ -57,16 +54,14 @@ public class BlurIndexReader extends BlurIndex {
     _shardContext = shardContext;
     _refresher = refresher;
     _closer = closer;
-    IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _tableContext.getAnalyzer());
-    conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5));
-    conf.setIndexDeletionPolicy(_tableContext.getIndexDeletionPolicy());
-    conf.setSimilarity(_tableContext.getSimilarity());
-    TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
-    mergePolicy.setUseCompoundFile(false);
 
     _open.set(true);
 
     if (!DirectoryReader.indexExists(directory)) {
+      LOG.info("Creating an empty index");
+      // if the directory is empty then create an empty index.
+      IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _tableContext.getAnalyzer());
+      conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5));
       new IndexWriter(directory, conf).close();
     }
     _indexReaderRef.set(DirectoryReader.open(directory));
@@ -107,12 +102,25 @@ public class BlurIndexReader extends BlurIndex {
 
   @Override
   public void optimize(int numberOfSegmentsPerShard) throws IOException {
-    // Do nothing
+    throw new RuntimeException("Read-only shard");
   }
 
   @Override
   public IndexSearcherClosable getIndexReader() throws IOException {
-    throw new RuntimeException("not implemented");
+    final DirectoryReader reader = _indexReaderRef.get();
+    reader.incRef();
+    return new IndexSearcherClosable(reader, null) {
+
+      @Override
+      public Directory getDirectory() {
+        return _directory;
+      }
+
+      @Override
+      public void close() throws IOException {
+        reader.decRef();
+      }
+    };
   }
 
   @Override
@@ -120,10 +128,4 @@ public class BlurIndexReader extends BlurIndex {
     return _isClosed;
   }
 
-  public IndexSearcher getSearcher() {
-    IndexReader indexReader = _indexReaderRef.get();
-    indexReader.incRef();
-    return new IndexSearcher(indexReader);
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b73a6ace/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index 9adbc46..43310fb 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@ -35,6 +35,7 @@ import org.apache.blur.lucene.store.refcounter.DirectoryReferenceCounter;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
 import org.apache.blur.lucene.store.refcounter.IndexInputCloser;
 import org.apache.blur.server.IndexSearcherClosable;
+import org.apache.blur.server.IndexSearcherClosableNRT;
 import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
 import org.apache.blur.thrift.generated.Record;
@@ -97,7 +98,7 @@ public class BlurNRTIndex extends BlurIndex {
     _searcherFactory = new SearcherFactory() {
       @Override
       public IndexSearcher newSearcher(IndexReader reader) throws IOException {
-        return new IndexSearcherClosable(reader, searchExecutor, _nrtManagerRef, _directory);
+        return new IndexSearcherClosableNRT(reader, searchExecutor, _nrtManagerRef, _directory);
       }
     };
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b73a6ace/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java
index 64574d8..331a9ff 100644
--- a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java
+++ b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosable.java
@@ -1,34 +1,38 @@
 package org.apache.blur.server;
 
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.NRTManager;
 import org.apache.lucene.store.Directory;
 
-public class IndexSearcherClosable extends IndexSearcher implements Closeable {
+public abstract class IndexSearcherClosable extends IndexSearcher implements Closeable {
 
-  private final AtomicReference<NRTManager> _nrtManagerRef;
-  private final Directory _directory;
-
-  public IndexSearcherClosable(IndexReader r, ExecutorService executor, AtomicReference<NRTManager>
nrtManagerRef,
-      Directory directory) {
+  public IndexSearcherClosable(IndexReader r, ExecutorService executor) {
     super(r, executor);
-    _nrtManagerRef = nrtManagerRef;
-    _directory = directory;
   }
 
-  public Directory getDirectory() {
-    return _directory;
-  }
+  public abstract Directory getDirectory();
 
   @Override
-  public void close() throws IOException {
-    _nrtManagerRef.get().release(this);
-  }
+  public abstract void close() throws IOException;
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b73a6ace/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java
b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java
new file mode 100644
index 0000000..ddd28ab
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java
@@ -0,0 +1,49 @@
+package org.apache.blur.server;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.NRTManager;
+import org.apache.lucene.store.Directory;
+
+public class IndexSearcherClosableNRT extends IndexSearcherClosable {
+
+  private final AtomicReference<NRTManager> _nrtManagerRef;
+  private final Directory _directory;
+
+  public IndexSearcherClosableNRT(IndexReader r, ExecutorService executor, AtomicReference<NRTManager>
nrtManagerRef,
+      Directory directory) {
+    super(r, executor);
+    _nrtManagerRef = nrtManagerRef;
+    _directory = directory;
+  }
+  
+  @Override
+  public Directory getDirectory() {
+    return _directory;
+  }
+
+  @Override
+  public void close() throws IOException {
+    _nrtManagerRef.get().release(this);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b73a6ace/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java
b/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java
index c568cd1..da06232 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/indexserver/SafeModeTest.java
@@ -77,7 +77,8 @@ public class SafeModeTest {
     }
 
     for (AtomicReference<Throwable> t : errors) {
-      assertNull(t.get());
+      Throwable throwable = t.get();
+      assertNull(throwable == null ? null : throwable.getMessage(), throwable);
     }
 
     long oldest = -1;
@@ -91,7 +92,7 @@ public class SafeModeTest {
         newest = l;
       }
     }
-    assertTrue((newest - oldest) < TimeUnit.SECONDS.toMillis(1));
+    assertTrue("newest [" + newest + "] oldest [" + oldest + "]", (newest - oldest) <
TimeUnit.SECONDS.toMillis(5));
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b73a6ace/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
index f225cfc..76886e4 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
@@ -21,32 +21,34 @@ import static org.junit.Assert.assertEquals;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.Random;
+import java.util.UUID;
 import java.util.concurrent.ExecutorService;
 
 import org.apache.blur.concurrent.Executors;
+import org.apache.blur.index.IndexWriter;
 import org.apache.blur.lucene.store.refcounter.DirectoryReferenceFileGC;
 import org.apache.blur.lucene.store.refcounter.IndexInputCloser;
+import org.apache.blur.server.IndexSearcherClosable;
 import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
 import org.apache.blur.thrift.generated.AnalyzerDefinition;
-import org.apache.blur.thrift.generated.Column;
-import org.apache.blur.thrift.generated.Record;
-import org.apache.blur.thrift.generated.Row;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.util.Version;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
 public class BlurIndexReaderTest {
 
-  private static final File TMPDIR = new File("./target/tmp");
+  private static final File TMPDIR = new File("./target/tmp/BlurIndexReaderTest");
 
-  private BlurNRTIndex writer;
-  private Random random = new Random();
   private ExecutorService service;
   private File base;
   private Configuration configuration;
@@ -56,6 +58,10 @@ public class BlurIndexReaderTest {
   private SharedMergeScheduler mergeScheduler;
   private BlurIndexReader reader;
 
+  private BlurIndexRefresher refresher;
+  private BlurIndexCloser indexCloser;
+  private FSDirectory directory;
+
   @Before
   public void setup() throws IOException {
     TableContext.clear();
@@ -71,25 +77,23 @@ public class BlurIndexReaderTest {
 
     configuration = new Configuration();
     service = Executors.newThreadPool("test", 1);
-    
+
   }
 
   private void setupWriter(Configuration configuration, long refresh) throws IOException
{
     TableDescriptor tableDescriptor = new TableDescriptor();
     tableDescriptor.setName("test-table");
-    tableDescriptor.setTableUri(new File(base, "table-store").toURI().toString());
+    tableDescriptor.setTableUri(new File(base, "table-store-" + UUID.randomUUID().toString()).toURI().toString());
     tableDescriptor.setAnalyzerDefinition(new AnalyzerDefinition());
     tableDescriptor.putToTableProperties("blur.shard.time.between.refreshs", Long.toString(refresh));
     tableDescriptor.putToTableProperties("blur.shard.time.between.commits", Long.toString(1000));
-    
+
     TableContext tableContext = TableContext.create(tableDescriptor);
-    FSDirectory directory = FSDirectory.open(new File(base, "index"));
+    directory = FSDirectory.open(new File(tableDescriptor.getTableUri()));
 
     ShardContext shardContext = ShardContext.create(tableContext, "test-shard");
-
-    writer = new BlurNRTIndex(shardContext, mergeScheduler, closer, directory, gc, service);
-    BlurIndexRefresher refresher = new BlurIndexRefresher();
-    BlurIndexCloser indexCloser = new BlurIndexCloser();
+    refresher = new BlurIndexRefresher();
+    indexCloser = new BlurIndexCloser();
     refresher.init();
     indexCloser.init();
     reader = new BlurIndexReader(shardContext, directory, refresher, indexCloser);
@@ -97,7 +101,6 @@ public class BlurIndexReaderTest {
 
   @After
   public void tearDown() throws IOException {
-    writer.close();
     mergeScheduler.close();
     closer.close();
     gc.close();
@@ -120,25 +123,25 @@ public class BlurIndexReaderTest {
   @Test
   public void testBlurIndexWriter() throws IOException, InterruptedException {
     setupWriter(configuration, 1);
-    IndexSearcher searcher = reader.getSearcher();
-    writer.replaceRow(true, true, genRow());
-    Thread.sleep(1500);
-    assertEquals(0,searcher.getIndexReader().numDocs());
+    IndexSearcherClosable indexReader1 = reader.getIndexReader();
+    doWrite();
+    assertEquals(0, indexReader1.getIndexReader().numDocs());
+    indexReader1.close();
     reader.refresh();
-    assertEquals(1,reader.getSearcher().getIndexReader().numDocs());
+    IndexSearcherClosable indexReader2 = reader.getIndexReader();
+    assertEquals(1, indexReader2.getIndexReader().numDocs());
+    indexReader2.close();
   }
-  
-  private Row genRow() {
-    Row row = new Row();
-    row.setId(Long.toString(random.nextLong()));
-    Record record = new Record();
-    record.setFamily("testing");
-    record.setRecordId(Long.toString(random.nextLong()));
-    for (int i = 0; i < 10; i++) {
-      record.addToColumns(new Column("col" + i, Long.toString(random.nextLong())));
-    }
-    row.addToRecords(record);
-    return row;
+
+  private void doWrite() throws CorruptIndexException, LockObtainFailedException, IOException
{
+    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
+    IndexWriter writer = new IndexWriter(directory, conf);
+    writer.addDocument(getDoc());
+    writer.close();
+  }
+
+  private Document getDoc() {
+    return new Document();
   }
 
 }


Mime
View raw message