geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ladyva...@apache.org
Subject [15/51] [abbrv] geode git commit: GEODE-3128: Changed the batch size to 1000 when creating AEQFactory
Date Wed, 19 Jul 2017 18:04:33 GMT
GEODE-3128: Changed the batch size to 1000 when creating AEQFactory

	This closes #603


Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/96bc0ebf
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/96bc0ebf
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/96bc0ebf

Branch: refs/heads/feature/GEODE-3113
Commit: 96bc0ebf1cc5123bab2b743d3a8ae4c282d86d06
Parents: 68af1df
Author: David Anuta <david.r.anuta@gmail.com>
Authored: Mon Jun 19 08:55:34 2017 -0700
Committer: nabarun <nnag@pivotal.io>
Committed: Wed Jul 12 14:50:12 2017 -0700

----------------------------------------------------------------------
 .../cache/lucene/internal/LuceneIndexImpl.java  |   1 +
 .../lucene/internal/filesystem/FileSystem.java  |   1 +
 .../lucene/LuceneQueriesIntegrationTest.java    |  17 ++-
 .../cache/lucene/MixedObjectIndexDUnitTest.java |  20 +--
 .../IndexRepositoryImplPerformanceTest.java     |  58 +++++----
 .../LuceneIndexMemoryOverheadTest.java          | 129 +++++++++++++++++++
 6 files changed, 190 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/geode/blob/96bc0ebf/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java
b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java
index 3393bcf..31efd8a 100644
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java
@@ -175,6 +175,7 @@ public abstract class LuceneIndexImpl implements InternalLuceneIndex {
     }
     factory.setMaximumQueueMemory(1000);
     factory.setDispatcherThreads(10);
+    factory.setBatchSize(1000);
     factory.setIsMetaQueue(true);
     if (attributes.getDataPolicy().withPersistence()) {
       factory.setPersistent(true);

http://git-wip-us.apache.org/repos/asf/geode/blob/96bc0ebf/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/filesystem/FileSystem.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/filesystem/FileSystem.java
b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/filesystem/FileSystem.java
index 164955f..3facef0 100644
--- a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/filesystem/FileSystem.java
+++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/filesystem/FileSystem.java
@@ -69,6 +69,7 @@ public class FileSystem {
       throw new IOException("File exists.");
     }
     stats.incFileCreates(1);
+
     // TODO unlock region ?
     return file;
   }

http://git-wip-us.apache.org/repos/asf/geode/blob/96bc0ebf/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesIntegrationTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesIntegrationTest.java
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesIntegrationTest.java
index 685d61c..3c0b10d 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesIntegrationTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesIntegrationTest.java
@@ -14,8 +14,13 @@
  */
 package org.apache.geode.cache.lucene;
 
-import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.*;
-import static org.junit.Assert.*;
+import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.DEFAULT_FIELD;
+import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.IntRangeQueryProvider;
+import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.verifyQueryKeyAndValues;
+import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.verifyQueryKeys;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -26,9 +31,6 @@ import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
-import org.apache.geode.cache.CacheLoader;
-import org.apache.geode.cache.CacheLoaderException;
-import org.apache.geode.cache.LoaderHelper;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
@@ -44,6 +46,9 @@ import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.ExpectedException;
 
+import org.apache.geode.cache.CacheLoader;
+import org.apache.geode.cache.CacheLoaderException;
+import org.apache.geode.cache.LoaderHelper;
 import org.apache.geode.cache.Region;
 import org.apache.geode.cache.RegionShortcut;
 import org.apache.geode.cache.lucene.test.TestObject;
@@ -59,7 +64,7 @@ public class LuceneQueriesIntegrationTest extends LuceneIntegrationTest
{
   @Rule
   public ExpectedException thrown = ExpectedException.none();
   private static final String INDEX_NAME = "index";
-  protected static final String REGION_NAME = "index";
+  public static final String REGION_NAME = "index";
   private Region region;
 
   @Test()

http://git-wip-us.apache.org/repos/asf/geode/blob/96bc0ebf/geode-lucene/src/test/java/org/apache/geode/cache/lucene/MixedObjectIndexDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/MixedObjectIndexDUnitTest.java
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/MixedObjectIndexDUnitTest.java
index 949ce25..90bf44a 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/MixedObjectIndexDUnitTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/MixedObjectIndexDUnitTest.java
@@ -15,22 +15,26 @@
 package org.apache.geode.cache.lucene;
 
 
-import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.*;
+import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.FloatRangeQueryProvider;
+import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.INDEX_NAME;
+import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.IntRangeQueryProvider;
+import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.REGION_NAME;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-import org.apache.geode.cache.Region;
-import org.apache.geode.test.dunit.SerializableRunnableIF;
-import org.apache.geode.test.junit.categories.DistributedTest;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
-
 import java.io.Serializable;
 import java.util.List;
 import java.util.stream.IntStream;
+
 import junitparams.JUnitParamsRunner;
 import junitparams.Parameters;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import org.apache.geode.cache.Region;
+import org.apache.geode.test.dunit.SerializableRunnableIF;
+import org.apache.geode.test.junit.categories.DistributedTest;
 
 @Category(DistributedTest.class)
 @RunWith(JUnitParamsRunner.class)

http://git-wip-us.apache.org/repos/asf/geode/blob/96bc0ebf/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplPerformanceTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplPerformanceTest.java
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplPerformanceTest.java
index 46425b9..4b321a5 100644
--- a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplPerformanceTest.java
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplPerformanceTest.java
@@ -15,20 +15,20 @@
 
 package org.apache.geode.cache.lucene.internal.repository;
 
-import org.apache.geode.DataSerializable;
-import org.apache.geode.DataSerializer;
-import org.apache.geode.cache.*;
-import org.apache.geode.cache.asyncqueue.AsyncEventQueue;
-import org.apache.geode.cache.lucene.*;
-import org.apache.geode.cache.lucene.internal.LuceneIndexStats;
-import org.apache.geode.cache.lucene.internal.LuceneServiceImpl;
-import org.apache.geode.cache.lucene.internal.directory.RegionDirectory;
-import org.apache.geode.cache.lucene.internal.distributed.TopEntriesCollector;
-import org.apache.geode.cache.lucene.internal.filesystem.ChunkKey;
-import org.apache.geode.cache.lucene.internal.filesystem.File;
-import org.apache.geode.cache.lucene.internal.filesystem.FileSystemStats;
-import org.apache.geode.cache.lucene.internal.repository.serializer.HeterogeneousLuceneSerializer;
-import org.apache.geode.test.junit.categories.PerformanceTest;
+import static org.apache.geode.distributed.ConfigurationProperties.LOG_LEVEL;
+import static org.apache.geode.distributed.ConfigurationProperties.MCAST_PORT;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field.Store;
@@ -45,14 +45,28 @@ import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.geode.distributed.ConfigurationProperties.*;
+import org.apache.geode.DataSerializable;
+import org.apache.geode.DataSerializer;
+import org.apache.geode.cache.Cache;
+import org.apache.geode.cache.CacheFactory;
+import org.apache.geode.cache.PartitionAttributesFactory;
+import org.apache.geode.cache.Region;
+import org.apache.geode.cache.RegionShortcut;
+import org.apache.geode.cache.asyncqueue.AsyncEventQueue;
+import org.apache.geode.cache.lucene.LuceneIndex;
+import org.apache.geode.cache.lucene.LuceneQuery;
+import org.apache.geode.cache.lucene.LuceneQueryException;
+import org.apache.geode.cache.lucene.LuceneQueryProvider;
+import org.apache.geode.cache.lucene.LuceneService;
+import org.apache.geode.cache.lucene.LuceneServiceProvider;
+import org.apache.geode.cache.lucene.PageableLuceneQueryResults;
+import org.apache.geode.cache.lucene.internal.LuceneIndexStats;
+import org.apache.geode.cache.lucene.internal.LuceneServiceImpl;
+import org.apache.geode.cache.lucene.internal.directory.RegionDirectory;
+import org.apache.geode.cache.lucene.internal.distributed.TopEntriesCollector;
+import org.apache.geode.cache.lucene.internal.filesystem.FileSystemStats;
+import org.apache.geode.cache.lucene.internal.repository.serializer.HeterogeneousLuceneSerializer;
+import org.apache.geode.test.junit.categories.PerformanceTest;
 
 
 /**

http://git-wip-us.apache.org/repos/asf/geode/blob/96bc0ebf/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/LuceneIndexMemoryOverheadTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/LuceneIndexMemoryOverheadTest.java
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/LuceneIndexMemoryOverheadTest.java
new file mode 100644
index 0000000..28047ce
--- /dev/null
+++ b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/LuceneIndexMemoryOverheadTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information
regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version
2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain
a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express
+ * or implied. See the License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.geode.cache.lucene.internal.repository;
+
+
+import static org.apache.geode.cache.lucene.LuceneQueriesIntegrationTest.REGION_NAME;
+import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.INDEX_NAME;
+
+import java.nio.charset.Charset;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.logging.log4j.Logger;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.springframework.shell.converters.AvailableCommandsConverter;
+
+import org.apache.geode.cache.Region;
+import org.apache.geode.cache.RegionShortcut;
+import org.apache.geode.cache.lucene.LuceneIntegrationTest;
+import org.apache.geode.cache.lucene.test.TestObject;
+import org.apache.geode.cache.partition.PartitionRegionHelper;
+import org.apache.geode.internal.cache.InternalCache;
+import org.apache.geode.internal.cache.TombstoneService;
+import org.apache.geode.internal.logging.LogService;
+import org.apache.geode.internal.size.ObjectGraphSizer;
+import org.apache.geode.test.junit.categories.PerformanceTest;
+
+@Category(PerformanceTest.class)
+@Ignore("Tests have no assertions")
+public class LuceneIndexMemoryOverheadTest extends LuceneIntegrationTest {
+  private static int NUM_BATCHES = 30;
+  private static int NUM_ENTRIES = 10000;
+  private Random random = new Random(0);
+
+  private static final Logger logger = LogService.getLogger();
+  private Callable flush =
+      () -> luceneService.waitUntilFlushed(INDEX_NAME, REGION_NAME, 60000, TimeUnit.MILLISECONDS);
+  protected static ObjectGraphSizer.ObjectFilter filter =
+      (parent, object) -> !(object instanceof AvailableCommandsConverter);
+
+  @Test
+  public void perEntryNonIndexMemoryWithTombstoneGC() throws Exception {
+    measureOverhead(((InternalCache) cache).getTombstoneService(), () -> true);
+  }
+
+  @Test
+  public void perEntryNonIndexMemoryWithoutTombstoneGC() throws Exception {
+    measureOverhead(null, () -> true);
+  }
+
+  @Test
+  public void perEntryIndexMemoryWithTombstoneGC() throws Exception {
+    luceneService.createIndexFactory().setFields("field1", "field2").create(INDEX_NAME,
+        REGION_NAME);
+
+    measureOverhead(((InternalCache) cache).getTombstoneService(), flush);
+  }
+
+  @Test
+  public void perEntryIndexMemoryWithoutTombstoneGC() throws Exception {
+    luceneService.createIndexFactory().setFields("field1", "field2").create(INDEX_NAME,
+        REGION_NAME);
+
+    measureOverhead(null, flush);
+  }
+
+  private void waitForFlush(Callable flush) throws Exception {
+    while (flush.call().equals(false)) {
+      logger.info("Flush returned false. Waiting for flush to return true...");
+    }
+  }
+
+  private void measureOverhead(TombstoneService service, Callable flush) throws Exception
{
+
+    Region region = cache.createRegionFactory(RegionShortcut.PARTITION).create(REGION_NAME);
+    PartitionRegionHelper.assignBucketsToPartitions(region);
+
+    long emptySize = ObjectGraphSizer.size(cache, filter, false);
+    waitForFlush(flush);
+
+    long size;
+    int numEntries = 0;
+
+    for (int batch = 0; batch < NUM_BATCHES; batch++) {
+
+      size =
+          addEntriesAndGetMemorySize(region, numEntries, NUM_ENTRIES, service, flush) - emptySize;
+      numEntries = region.size();
+
+      logger.info(
+          numEntries + " entries take up " + size + ", per entry overhead " + size / numEntries);
+    }
+  }
+
+  private long addEntriesAndGetMemorySize(Region region, int start, int count,
+      TombstoneService service, Callable flush) throws Exception {
+    int c = 0;
+    for (int i = start; i < start + count; i++) {
+      byte[] field1 = new byte[3];
+      byte[] field2 = new byte[5];
+      random.nextBytes(field1);
+      random.nextBytes(field2);
+      TestObject test = new TestObject(new String(field1, Charset.forName("ASCII")),
+          new String(field2, Charset.forName("ASCII")));
+      region.put(i, test);
+    }
+
+    waitForFlush(flush);
+    if (service != null)
+      service.forceBatchExpirationForTests((int) service.getScheduledTombstoneCount());
+
+    return ObjectGraphSizer.size(cache, filter, false);
+  }
+}


Mime
View raw message