geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From n...@apache.org
Subject geode git commit: GEODE-1733: Lucene stats are displayed correctly.
Date Mon, 16 Jan 2017 17:54:57 GMT
Repository: geode
Updated Branches:
  refs/heads/develop a4f38f33a -> f1cebc2f0


GEODE-1733: Lucene stats are displayed correctly.

	* Lucene stats are displayed correctly when rebalancing occurs or a server is reinstantiated
	* Listener added which creates the LuceneIndexRepository when a bucket becomes primary.
	* LuceneIndexRepository no longer being created lazily.


Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/f1cebc2f
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/f1cebc2f
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/f1cebc2f

Branch: refs/heads/develop
Commit: f1cebc2f085807d48adf58d7ad0fbebced49e043
Parents: a4f38f3
Author: nabarun <nnag@pivotal.io>
Authored: Wed Jan 4 18:07:03 2017 -0800
Committer: nabarun <nnag@pivotal.io>
Committed: Mon Jan 16 09:51:42 2017 -0800

----------------------------------------------------------------------
 .../LuceneIndexForPartitionedRegion.java        | 27 ++++++++----
 .../internal/LucenePrimaryBucketListener.java   | 43 ++++++++++++++++++++
 .../repository/IndexRepositoryImpl.java         |  3 +-
 .../LuceneIndexForPartitionedRegionTest.java    | 18 ++++----
 .../IndexRepositoryImplJUnitTest.java           |  9 +++-
 5 files changed, 81 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/geode/blob/f1cebc2f/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java
b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java
index c760726..5968912 100644
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java
@@ -33,6 +33,7 @@ import org.apache.geode.cache.lucene.internal.filesystem.File;
 import org.apache.geode.cache.lucene.internal.filesystem.FileSystemStats;
 import org.apache.geode.cache.lucene.internal.repository.RepositoryManager;
 import org.apache.geode.cache.lucene.internal.repository.serializer.HeterogeneousLuceneSerializer;
+import org.apache.geode.cache.partition.PartitionListener;
 import org.apache.geode.internal.cache.GemFireCacheImpl;
 import org.apache.geode.internal.cache.PartitionedRegion;
 
@@ -78,17 +79,22 @@ public class LuceneIndexForPartitionedRegion extends LuceneIndexImpl {
 
     // create PR chunkRegion, but not to create its buckets for now
     final String chunkRegionName = createChunkRegionName();
+
+    // we will create RegionDirectories on the fly when data comes in
+    HeterogeneousLuceneSerializer mapper = new HeterogeneousLuceneSerializer(getFieldNames());
+    PartitionedRepositoryManager partitionedRepositoryManager =
+        new PartitionedRepositoryManager(this, mapper);
+    LucenePrimaryBucketListener lucenePrimaryBucketListener =
+        new LucenePrimaryBucketListener(partitionedRepositoryManager);
     if (!chunkRegionExists(chunkRegionName)) {
       chunkRegion = createChunkRegion(regionShortCut, fileRegionName, partitionAttributes,
-          chunkRegionName, regionAttributes);
+          chunkRegionName, regionAttributes, lucenePrimaryBucketListener);
     }
     fileSystemStats.setFileSupplier(() -> (int) getFileRegion().getLocalSize());
     fileSystemStats.setChunkSupplier(() -> (int) getChunkRegion().getLocalSize());
     fileSystemStats.setBytesSupplier(() -> getChunkRegion().getPrStats().getDataStoreBytesInUse());
 
-    // we will create RegionDirectories on the fly when data comes in
-    HeterogeneousLuceneSerializer mapper = new HeterogeneousLuceneSerializer(getFieldNames());
-    return new PartitionedRepositoryManager(this, mapper);
+    return partitionedRepositoryManager;
   }
 
   public PartitionedRegion getFileRegion() {
@@ -110,7 +116,7 @@ public class LuceneIndexForPartitionedRegion extends LuceneIndexImpl {
   Region createFileRegion(final RegionShortcut regionShortCut, final String fileRegionName,
       final PartitionAttributes partitionAttributes, final RegionAttributes regionAttributes)
{
     return createRegion(fileRegionName, regionShortCut, this.regionPath, partitionAttributes,
-        regionAttributes);
+        regionAttributes, null);
   }
 
   public String createFileRegionName() {
@@ -123,9 +129,10 @@ public class LuceneIndexForPartitionedRegion extends LuceneIndexImpl
{
 
   Region<ChunkKey, byte[]> createChunkRegion(final RegionShortcut regionShortCut,
       final String fileRegionName, final PartitionAttributes partitionAttributes,
-      final String chunkRegionName, final RegionAttributes regionAttributes) {
+      final String chunkRegionName, final RegionAttributes regionAttributes,
+      final PartitionListener lucenePrimaryBucketListener) {
     return createRegion(chunkRegionName, regionShortCut, fileRegionName, partitionAttributes,
-        regionAttributes);
+        regionAttributes, lucenePrimaryBucketListener);
   }
 
   public String createChunkRegionName() {
@@ -142,9 +149,13 @@ public class LuceneIndexForPartitionedRegion extends LuceneIndexImpl
{
 
   protected <K, V> Region<K, V> createRegion(final String regionName,
       final RegionShortcut regionShortCut, final String colocatedWithRegionName,
-      final PartitionAttributes partitionAttributes, final RegionAttributes regionAttributes)
{
+      final PartitionAttributes partitionAttributes, final RegionAttributes regionAttributes,
+      PartitionListener lucenePrimaryBucketListener) {
     PartitionAttributesFactory partitionAttributesFactory =
         new PartitionAttributesFactory<String, File>();
+    if (lucenePrimaryBucketListener != null) {
+      partitionAttributesFactory.addPartitionListener(lucenePrimaryBucketListener);
+    }
     partitionAttributesFactory.setColocatedWith(colocatedWithRegionName);
     configureLuceneRegionAttributesFactory(partitionAttributesFactory, partitionAttributes);
 

http://git-wip-us.apache.org/repos/asf/geode/blob/f1cebc2f/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LucenePrimaryBucketListener.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LucenePrimaryBucketListener.java
b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LucenePrimaryBucketListener.java
new file mode 100644
index 0000000..386a8b0
--- /dev/null
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LucenePrimaryBucketListener.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information
regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version
2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain
a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express
+ * or implied. See the License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.geode.cache.lucene.internal;
+
+import org.apache.geode.GemFireException;
+import org.apache.geode.cache.execute.FunctionException;
+import org.apache.geode.cache.partition.PartitionListenerAdapter;
+import org.apache.geode.internal.cache.BucketNotFoundException;
+import org.apache.geode.internal.logging.LogService;
+import org.apache.logging.log4j.Logger;
+
+public class LucenePrimaryBucketListener extends PartitionListenerAdapter {
+  private static final Logger logger = LogService.getLogger();
+  private PartitionedRepositoryManager lucenePartitionRepositoryManager;
+
+  public LucenePrimaryBucketListener(PartitionedRepositoryManager partitionedRepositoryManager)
{
+    lucenePartitionRepositoryManager = partitionedRepositoryManager;
+  }
+
+  @Override
+  public void afterPrimary(int bucketId) {
+    try {
+      lucenePartitionRepositoryManager.getRepository(bucketId);
+    } catch (BucketNotFoundException e) {
+      logger.warn(
+          "Index repository could not be created when index chunk region bucket became primary.
"
+              + "Deferring index repository to be created lazily during lucene query execution."
+              + e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/geode/blob/f1cebc2f/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImpl.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImpl.java
b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImpl.java
index 572b5ea..4e86eb5 100644
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImpl.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImpl.java
@@ -20,6 +20,7 @@ import org.apache.geode.cache.lucene.internal.LuceneIndexStats;
 import org.apache.geode.cache.lucene.internal.repository.serializer.LuceneSerializer;
 import org.apache.geode.cache.lucene.internal.repository.serializer.SerializerUtil;
 import org.apache.geode.distributed.internal.DistributionConfig;
+import org.apache.geode.internal.cache.BucketRegion;
 import org.apache.geode.internal.logging.LogService;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.document.Document;
@@ -163,7 +164,7 @@ public class IndexRepositoryImpl implements IndexRepository {
   private class DocumentCountSupplier implements IntSupplier {
     @Override
     public int getAsInt() {
-      if (isClosed()) {
+      if (isClosed() || !((BucketRegion) userRegion).getBucketAdvisor().isPrimary()) {
         stats.removeDocumentsSupplier(this);
         return 0;
       }

http://git-wip-us.apache.org/repos/asf/geode/blob/f1cebc2f/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegionTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegionTest.java
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegionTest.java
index 5c0c0a0..15235eb 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegionTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegionTest.java
@@ -245,7 +245,7 @@ public class LuceneIndexForPartitionedRegionTest {
     index.setSearchableFields(new String[] {"field"});
     LuceneIndexForPartitionedRegion spy = spy(index);
     doReturn(null).when(spy).createFileRegion(any(), any(), any(), any());
-    doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any());
+    doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any(), any());
     doReturn(null).when(spy).createAEQ(eq(region));
     spy.initialize();
     return spy;
@@ -264,7 +264,7 @@ public class LuceneIndexForPartitionedRegionTest {
     LuceneIndexForPartitionedRegion spy = setupSpy(region, index);
 
     verify(spy).createChunkRegion(eq(RegionShortcut.PARTITION), eq(index.createFileRegionName()),
-        any(), eq(index.createChunkRegionName()), any());
+        any(), eq(index.createChunkRegionName()), any(), any());
   }
 
   @Test
@@ -298,7 +298,7 @@ public class LuceneIndexForPartitionedRegionTest {
         partitionAttributes, regionAttributes);
     String fileRegionName = index.createFileRegionName();
     verify(indexSpy).createRegion(fileRegionName, RegionShortcut.PARTITION, regionPath,
-        partitionAttributes, regionAttributes);
+        partitionAttributes, regionAttributes, null);
     verify(cache).createVMRegion(eq(fileRegionName), any(), any());
   }
 
@@ -316,9 +316,9 @@ public class LuceneIndexForPartitionedRegionTest {
     String chunkRegionName = index.createChunkRegionName();
     String fileRegionName = index.createFileRegionName();
     indexSpy.createChunkRegion(RegionShortcut.PARTITION, fileRegionName, partitionAttributes,
-        chunkRegionName, regionAttributes);
+        chunkRegionName, regionAttributes, null);
     verify(indexSpy).createRegion(chunkRegionName, RegionShortcut.PARTITION, fileRegionName,
-        partitionAttributes, regionAttributes);
+        partitionAttributes, regionAttributes, null);
     verify(cache).createVMRegion(eq(chunkRegionName), any(), any());
   }
 
@@ -335,12 +335,12 @@ public class LuceneIndexForPartitionedRegionTest {
     index.setSearchableFields(new String[] {"field"});
     LuceneIndexForPartitionedRegion spy = spy(index);
     doReturn(null).when(spy).createFileRegion(any(), any(), any(), any());
-    doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any());
+    doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any(), any());
     doReturn(null).when(spy).createAEQ(any());
     spy.initialize();
 
     verify(spy).createChunkRegion(eq(RegionShortcut.PARTITION_PERSISTENT),
-        eq(index.createFileRegionName()), any(), eq(index.createChunkRegionName()), any());
+        eq(index.createFileRegionName()), any(), eq(index.createChunkRegionName()), any(),
any());
   }
 
   @Test
@@ -356,7 +356,7 @@ public class LuceneIndexForPartitionedRegionTest {
     index.setSearchableFields(new String[] {"field"});
     LuceneIndexForPartitionedRegion spy = spy(index);
     doReturn(null).when(spy).createFileRegion(any(), any(), any(), any());
-    doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any());
+    doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any(), any());
     doReturn(null).when(spy).createAEQ(any());
     spy.initialize();
 
@@ -377,7 +377,7 @@ public class LuceneIndexForPartitionedRegionTest {
     index.setSearchableFields(new String[] {"field"});
     LuceneIndexForPartitionedRegion spy = spy(index);
     doReturn(null).when(spy).createFileRegion(any(), any(), any(), any());
-    doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any());
+    doReturn(null).when(spy).createChunkRegion(any(), any(), any(), any(), any(), any());
     doReturn(null).when(spy).createAEQ(any());
     spy.initialize();
     spy.initialize();

http://git-wip-us.apache.org/repos/asf/geode/blob/f1cebc2f/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplJUnitTest.java
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplJUnitTest.java
index dd52258..7426fa5 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplJUnitTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplJUnitTest.java
@@ -27,6 +27,8 @@ import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.IntSupplier;
 
+import org.apache.geode.internal.cache.BucketAdvisor;
+import org.apache.geode.internal.cache.BucketRegion;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -75,7 +77,12 @@ public class IndexRepositoryImplJUnitTest {
     String[] indexedFields = new String[] {"s", "i", "l", "d", "f", "s2", "missing"};
     mapper = new HeterogeneousLuceneSerializer(indexedFields);
     region = Mockito.mock(Region.class);
-    userRegion = Mockito.mock(Region.class);
+    userRegion = Mockito.mock(BucketRegion.class);
+    BucketAdvisor bucketAdvisor = Mockito.mock(BucketAdvisor.class);
+    Mockito.when(bucketAdvisor.isPrimary()).thenReturn(true);
+    Mockito.when(((BucketRegion) userRegion).getBucketAdvisor()).thenReturn(bucketAdvisor);
+
+    Mockito.when(((BucketRegion) userRegion).getBucketAdvisor().isPrimary()).thenReturn(true);
     stats = Mockito.mock(LuceneIndexStats.class);
     Mockito.when(userRegion.isDestroyed()).thenReturn(false);
     repo = new IndexRepositoryImpl(region, writer, mapper, stats, userRegion);


Mime
View raw message