incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: BLUR-208: Upgrade blur to lucene 4.4 version Code is in broken state
Date Mon, 14 Oct 2013 13:18:52 GMT
Updated Branches:
  refs/heads/0.3.0-lucene-upgrade [created] 7d8bc4220


BLUR-208: Upgrade blur to lucene 4.4 version Code is in broken state

Signed-off-by: Aaron McCurry <amccurry@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/7d8bc422
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/7d8bc422
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/7d8bc422

Branch: refs/heads/0.3.0-lucene-upgrade
Commit: 7d8bc4220816a59c2b713c01a512c8d3ec56e309
Parents: fda400d
Author: Saurabh Gupta <saurabh.b85@gmail.com>
Authored: Sun Oct 13 07:37:22 2013 +0100
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Oct 14 09:18:31 2013 -0400

----------------------------------------------------------------------
 .../blur/manager/writer/BlurNRTIndex.java       | 43 +++++++++++---------
 .../blur/manager/writer/IndexImporter.java      |  2 +-
 .../manager/writer/TransactionRecorder.java     |  2 +-
 .../blur/server/IndexSearcherClosableNRT.java   |  6 +--
 .../manager/writer/BlurIndexReaderTest.java     |  2 +-
 .../blur/manager/writer/IndexImporterTest.java  |  2 +-
 .../utils/TableShardCountCollapserTest.java     |  4 +-
 .../org/apache/blur/mapreduce/BlurReducer.java  |  4 --
 .../blur/mapreduce/lib/BlurOutputFormat.java    |  2 -
 .../blur/lucene/search/FairSimilarity.java      |  9 ++++
 .../blur/lucene/LuceneVersionConstant.java      |  2 +-
 pom.xml                                         |  2 +-
 12 files changed, 42 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index d593089..cd7df07 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@ -63,12 +63,12 @@ import org.apache.lucene.index.IndexDeletionPolicy;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.SnapshotDeletionPolicy;
-import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.lucene.index.TrackingIndexWriter;
+import org.apache.lucene.search.ControlledRealTimeReopenThread;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.NRTManager;
-import org.apache.lucene.search.NRTManager.TrackingIndexWriter;
-import org.apache.lucene.search.NRTManagerReopenThread;
+import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.search.SearcherManager;
 import org.apache.lucene.store.Directory;
 
 public class BlurNRTIndex extends BlurIndex {
@@ -78,13 +78,13 @@ public class BlurNRTIndex extends BlurIndex {
   private static final String SNAPSHOTS_FOLDER_NAME = "snapshots";
   private static final String SNAPSHOTS_TMPFILE_EXTENSION = ".tmp";
 
-  private final AtomicReference<NRTManager> _nrtManagerRef = new AtomicReference<NRTManager>();
+  private final AtomicReference<SearcherManager> _nrtManagerRef = new AtomicReference<SearcherManager>();
   private final AtomicBoolean _isClosed = new AtomicBoolean();
   private final BlurIndexWriter _writer;
   private final Thread _committer;
   private final SearcherFactory _searcherFactory;
   private final Directory _directory;
-  private final NRTManagerReopenThread _refresher;
+  private final ControlledRealTimeReopenThread<IndexSearcher> _refresher;
   private final TableContext _tableContext;
   private final ShardContext _shardContext;
   private final TransactionRecorder _recorder;
@@ -110,15 +110,13 @@ public class BlurNRTIndex extends BlurIndex {
     SnapshotDeletionPolicy sdp;
     if (snapshotsDirectoryExists()) {
       // load existing snapshots
-      sdp = new SnapshotDeletionPolicy(_tableContext.getIndexDeletionPolicy(), loadExistingSnapshots());
+     sdp = new SnapshotDeletionPolicy(_tableContext.getIndexDeletionPolicy(), loadExistingSnapshots());
     } else {
       sdp = new SnapshotDeletionPolicy(_tableContext.getIndexDeletionPolicy());
-    }
+   }
     conf.setIndexDeletionPolicy(sdp);
     conf.setMergedSegmentWarmer(new FieldBasedWarmer(shardContext, _isClosed));
 
-    TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
-    mergePolicy.setUseCompoundFile(false);
     conf.setMergeScheduler(mergeScheduler.getMergeScheduler());
 
     DirectoryReferenceCounter referenceCounter = new DirectoryReferenceCounter(directory,
gc, closer);
@@ -142,7 +140,7 @@ public class BlurNRTIndex extends BlurIndex {
 
     _trackingWriter = new TrackingIndexWriter(_writer);
     _indexImporter = new IndexImporter(_trackingWriter, _lock, _shardContext, TimeUnit.SECONDS,
10);
-    _nrtManagerRef.set(new NRTManager(_trackingWriter, _searcherFactory, APPLY_ALL_DELETES));
+    _nrtManagerRef.set(new SearcherManager(_writer, APPLY_ALL_DELETES,_searcherFactory));
     // start commiter
 
     _committer = new Thread(new Committer());
@@ -152,7 +150,7 @@ public class BlurNRTIndex extends BlurIndex {
 
     // start refresher
     double targetMinStaleSec = _tableContext.getTimeBetweenRefreshs() / 1000.0;
-    _refresher = new NRTManagerReopenThread(getNRTManager(), targetMinStaleSec * 10, targetMinStaleSec);
+    _refresher = new ControlledRealTimeReopenThread<IndexSearcher>(_trackingWriter,getNRTManager(),
targetMinStaleSec * 10, targetMinStaleSec);
     _refresher.setName("Refresh Thread [" + _tableContext.getTable() + "/" + shardContext.getShard()
+ "]");
     _refresher.setDaemon(true);
     _refresher.start();
@@ -187,6 +185,7 @@ public class BlurNRTIndex extends BlurIndex {
         snapshots.put(snapshotName, segmentsFilename);
       }
     }
+    
     return snapshots;
   }
   
@@ -245,7 +244,7 @@ public class BlurNRTIndex extends BlurIndex {
     return indexSearcherClosable;
   }
 
-  private NRTManager getNRTManager() {
+  private ReferenceManager<IndexSearcher> getNRTManager() {
     return _nrtManagerRef.get();
   }
 
@@ -288,7 +287,7 @@ public class BlurNRTIndex extends BlurIndex {
       refresh();
     }
     if (waitToBeVisible && getNRTManager().getCurrentSearchingGen() < generation)
{
-      getNRTManager().waitForGeneration(generation);
+    	_refresher.waitForGeneration(generation);
     }
   }
 
@@ -340,12 +339,16 @@ public class BlurNRTIndex extends BlurIndex {
   @Override
   public void createSnapshot(String name) throws IOException {
     SnapshotDeletionPolicy snapshotter = getSnapshotter();
-    Map<String, String> existingSnapshots = snapshotter.getSnapshots();
-    if (existingSnapshots.containsKey(name)) {
-      LOG.error("A Snapshot already exists with the same name [{0}] on [{1}/{2}].", 
-          name, _tableContext.getTable(), _shardContext.getShard());
-      throw new IOException("A Snapshot already exists with the same name [" + name + "]
on " +
-      	"["+_tableContext.getTable()+"/"+_shardContext.getShard()+"].");
+    List<IndexCommit> snapshots = snapshotter.getSnapshots();
+    
+//    Map<String, String> existingSnapshots = snapshotter.getSnapshots();
+    for (IndexCommit indexCommit : snapshots) {
+	    if (indexCommit.getSegmentsFileName().equals(name)) {
+	      LOG.error("A Snapshot already exists with the same name [{0}] on [{1}/{2}].", 
+	          name, _tableContext.getTable(), _shardContext.getShard());
+	      throw new IOException("A Snapshot already exists with the same name [" + name + "]
on " +
+	      	"["+_tableContext.getTable()+"/"+_shardContext.getShard()+"].");
+	    }
     }
     _writer.commit();
     IndexCommit indexCommit = snapshotter.snapshot(name);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
index cbf5e91..34b6727 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
@@ -49,7 +49,7 @@ import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.NRTManager.TrackingIndexWriter;
+import org.apache.lucene.index.TrackingIndexWriter;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
index 605fe32..6a0bf31 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
@@ -58,7 +58,7 @@ import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.BlurIndexWriter;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.NRTManager.TrackingIndexWriter;
+import org.apache.lucene.index.TrackingIndexWriter;
 
 public class TransactionRecorder extends TimerTask implements Closeable {
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java
b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java
index ddd28ab..b6b2fe7 100644
--- a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java
+++ b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherClosableNRT.java
@@ -21,15 +21,15 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.NRTManager;
+import org.apache.lucene.search.SearcherManager;
 import org.apache.lucene.store.Directory;
 
 public class IndexSearcherClosableNRT extends IndexSearcherClosable {
 
-  private final AtomicReference<NRTManager> _nrtManagerRef;
+  private final AtomicReference<SearcherManager> _nrtManagerRef;
   private final Directory _directory;
 
-  public IndexSearcherClosableNRT(IndexReader r, ExecutorService executor, AtomicReference<NRTManager>
nrtManagerRef,
+  public IndexSearcherClosableNRT(IndexReader r, ExecutorService executor, AtomicReference<SearcherManager>
nrtManagerRef,
       Directory directory) {
     super(r, executor);
     _nrtManagerRef = nrtManagerRef;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
index 59316f3..f98f9d7 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/BlurIndexReaderTest.java
@@ -136,7 +136,7 @@ public class BlurIndexReaderTest {
   }
 
   private void doWrite() throws CorruptIndexException, LockObtainFailedException, IOException
{
-    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
+    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_44, new KeywordAnalyzer());
     BlurIndexWriter writer = new BlurIndexWriter(directory, conf);
     writer.addDocument(getDoc());
     writer.close();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
b/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
index e7724a5..cabf709 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/writer/IndexImporterTest.java
@@ -43,7 +43,7 @@ import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.search.NRTManager.TrackingIndexWriter;
+import org.apache.lucene.index.TrackingIndexWriter;
 import org.apache.lucene.store.Directory;
 import org.junit.After;
 import org.junit.Before;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java
b/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java
index 28bf718..5be4457 100644
--- a/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java
+++ b/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java
@@ -94,9 +94,7 @@ public class TableShardCountCollapserTest {
   private static void createShard(Configuration configuration, int i, Path path, int totalShardCount)
       throws IOException {
     HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, path);
-    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
-    TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy();
-    mergePolicy.setUseCompoundFile(false);
+    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_44, new KeywordAnalyzer());
     IndexWriter indexWriter = new IndexWriter(hdfsDirectory, conf);
 
     Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable,
IntWritable>();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
index 3669232..2e5773b 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java
@@ -304,8 +304,6 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate>
{
     if (optimize) {
       context.setStatus("Starting Copy-Optimize Phase");
       IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, _analyzer);
-      TieredMergePolicy policy = (TieredMergePolicy) conf.getMergePolicy();
-      policy.setUseCompoundFile(false);
       long s = System.currentTimeMillis();
       IndexWriter writer = new IndexWriter(getBiggerBuffers(destDirectory), conf);
       writer.addIndexes(reader);
@@ -461,8 +459,6 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate>
{
     IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, _analyzer);
     config.setSimilarity(new FairSimilarity());
     config.setRAMBufferSizeMB(_blurTask.getRamBufferSizeMB());
-    TieredMergePolicy mergePolicy = (TieredMergePolicy) config.getMergePolicy();
-    mergePolicy.setUseCompoundFile(false);
     _writer = new IndexWriter(_directory, config);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
index 2f59a20..725be30 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/BlurOutputFormat.java
@@ -399,8 +399,6 @@ public class BlurOutputFormat extends OutputFormat<Text, BlurMutate>
{
       Analyzer analyzer = _fieldManager.getAnalyzerForIndex();
 
       _conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer);
-      TieredMergePolicy mergePolicy = (TieredMergePolicy) _conf.getMergePolicy();
-      mergePolicy.setUseCompoundFile(false);
 
       _overFlowConf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, analyzer);
       _overFlowConf.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java b/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java
index 8c8b945..c3e0277 100644
--- a/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java
+++ b/blur-query/src/main/java/org/apache/blur/lucene/search/FairSimilarity.java
@@ -58,4 +58,13 @@ public class FairSimilarity extends TFIDFSimilarity {
 //    return 0;
   }
 
+  @Override
+  public float decodeNormValue(long norm) {
+  	return 1;
+  }
+
+  @Override
+  public long encodeNormValue(float f) {
+  	return 1;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java b/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java
index 20ae366..ab8cf85 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java
@@ -23,6 +23,6 @@ public class LuceneVersionConstant {
   /**
    * All Lucene Version references should refer to this constant.
    */
-  public static final Version LUCENE_VERSION = Version.LUCENE_43;
+  public static final Version LUCENE_VERSION = Version.LUCENE_44;
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7d8bc422/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 05e03b6..4d6c831 100644
--- a/pom.xml
+++ b/pom.xml
@@ -175,7 +175,7 @@ under the License.
 		<zookeeper.version>3.4.5</zookeeper.version>
 		<log4j.version>1.2.15</log4j.version>
 		<jersey.version>1.14</jersey.version>
-		<lucene.version>4.3.0</lucene.version>
+		<lucene.version>4.4.0</lucene.version>
 		<junit.version>4.7</junit.version>
 		<slf4j.version>1.6.1</slf4j.version>
 		<commons-cli.version>1.2</commons-cli.version>


Mime
View raw message