ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ptupit...@apache.org
Subject [19/23] ignite git commit: IGNITE-5884 Change default pageSize of page memory to 4KB
Date Fri, 08 Sep 2017 13:59:56 GMT
IGNITE-5884 Change default pageSize of page memory to 4KB


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a634e559
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a634e559
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a634e559

Branch: refs/heads/ignite-5896
Commit: a634e55956ab7b6b98dbe3f2de52c7758692b6e0
Parents: bacff9e
Author: Ivan Rakov <ivan.glukos@gmail.com>
Authored: Fri Sep 8 12:11:44 2017 +0300
Committer: Andrey Gura <agura@apache.org>
Committed: Fri Sep 8 12:11:44 2017 +0300

----------------------------------------------------------------------
 .../configuration/MemoryConfiguration.java      |   4 +-
 .../GridCacheDatabaseSharedManager.java         |  88 +++++++++--
 .../IgniteCacheDatabaseSharedManager.java       |  11 ++
 .../cache/persistence/file/FilePageStore.java   |   2 +-
 .../ignite/internal/util/IgniteUtils.java       |  41 ++++++
 ...faultPageSizeBackwardsCompatibilityTest.java | 147 +++++++++++++++++++
 .../ignite/testsuites/IgnitePdsTestSuite.java   |   3 +
 7 files changed, 280 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/a634e559/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java
b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java
index 8b59c4f..9ba26c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java
@@ -81,7 +81,7 @@ public class MemoryConfiguration implements Serializable {
     private static final long DFLT_SYS_CACHE_MAX_SIZE = 100 * 1024 * 1024;
 
     /** Default memory page size. */
-    public static final int DFLT_PAGE_SIZE = 2 * 1024;
+    public static final int DFLT_PAGE_SIZE = 4 * 1024;
 
     /** This name is assigned to default MemoryPolicy if no user-defined default MemPlc is
specified */
     public static final String DFLT_MEM_PLC_DEFAULT_NAME = "default";
@@ -93,7 +93,7 @@ public class MemoryConfiguration implements Serializable {
     private long sysCacheMaxSize = DFLT_SYS_CACHE_MAX_SIZE;
 
     /** Memory page size. */
-    private int pageSize = DFLT_PAGE_SIZE;
+    private int pageSize;
 
     /** Concurrency level. */
     private int concLvl;

http://git-wip-us.apache.org/repos/asf/ignite/blob/a634e559/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 4db0bf8..5e84048 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -27,6 +27,7 @@ import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.nio.channels.OverlappingFileLockException;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
@@ -102,6 +103,8 @@ import org.apache.ignite.internal.processors.cache.StoredCacheData;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
@@ -117,6 +120,7 @@ import org.apache.ignite.internal.processors.port.GridPortRecord;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.GridMultiCollectionWrapper;
 import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.future.CountDownFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridInClosure3X;
@@ -310,19 +314,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
         lockWaitTime = persistenceCfg.getLockWaitTime();
 
-        final int pageSize = cfg.getMemoryConfiguration().getPageSize();
-
-        threadBuf = new ThreadLocal<ByteBuffer>() {
-            /** {@inheritDoc} */
-            @Override protected ByteBuffer initialValue() {
-                ByteBuffer tmpWriteBuf = ByteBuffer.allocateDirect(pageSize);
-
-                tmpWriteBuf.order(ByteOrder.nativeOrder());
-
-                return tmpWriteBuf;
-            }
-        };
-
         persStoreMetrics = new PersistenceMetricsImpl(
             persistenceCfg.isMetricsEnabled(),
             persistenceCfg.getRateTimeInterval(),
@@ -356,6 +347,17 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     @Override protected void start0() throws IgniteCheckedException {
         super.start0();
 
+        threadBuf = new ThreadLocal<ByteBuffer>() {
+            /** {@inheritDoc} */
+            @Override protected ByteBuffer initialValue() {
+                ByteBuffer tmpWriteBuf = ByteBuffer.allocateDirect(pageSize());
+
+                tmpWriteBuf.order(ByteOrder.nativeOrder());
+
+                return tmpWriteBuf;
+            }
+        };
+
         snapshotMgr = cctx.snapshot();
 
         if (!cctx.kernalContext().clientNode()) {
@@ -701,6 +703,66 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
                 "because Ignite Native Persistence is enabled");
     }
 
+    /** {@inheritDoc} */
+    @Override protected void checkPageSize(MemoryConfiguration memCfg) {
+        if (memCfg.getPageSize() == 0) {
+            try {
+                assert cctx.pageStore() instanceof FilePageStoreManager :
+                    "Invalid page store manager was created: " + cctx.pageStore();
+
+                Path anyIdxPartFile = IgniteUtils.searchFileRecursively(
+                    ((FilePageStoreManager)cctx.pageStore()).workDir().toPath(), FilePageStoreManager.INDEX_FILE_NAME);
+
+                if (anyIdxPartFile != null) {
+                    memCfg.setPageSize(resolvePageSizeFromPartitionFile(anyIdxPartFile));
+
+                    return;
+                }
+            }
+            catch (IgniteCheckedException | IOException | IllegalArgumentException e) {
+                U.quietAndWarn(log, "Attempt to resolve pageSize from store files failed:
" + e.getMessage());
+
+                U.quietAndWarn(log, "Default page size will be used: " + MemoryConfiguration.DFLT_PAGE_SIZE
+ " bytes");
+            }
+
+            memCfg.setPageSize(MemoryConfiguration.DFLT_PAGE_SIZE);
+        }
+    }
+
+    /**
+     * @param partFile Partition file.
+     */
+    private int resolvePageSizeFromPartitionFile(Path partFile) throws IOException, IgniteCheckedException
{
+        try (FileIO fileIO = persistenceCfg.getFileIOFactory().create(partFile.toFile(),
"r")) {
+            int minimalHdr = FilePageStore.HEADER_SIZE;
+
+            if (fileIO.size() < minimalHdr)
+                throw new IgniteCheckedException("Partition file is too small: " + partFile);
+
+            ByteBuffer hdr = ByteBuffer.allocate(minimalHdr).order(ByteOrder.LITTLE_ENDIAN);
+
+            while (hdr.remaining() > 0)
+                fileIO.read(hdr);
+
+            hdr.rewind();
+
+            hdr.getLong(); // Read signature.
+
+            hdr.getInt(); // Read version.
+
+            hdr.get(); // Read type.
+
+            int pageSize = hdr.getInt();
+
+            if (pageSize == 2048) {
+                U.quietAndWarn(log, "You are currently using persistent store with 2K pages
(MemoryConfiguration#" +
+                    "pageSize). If you use SSD disk, consider migrating to 4K pages for better
IO performance.");
+            }
+
+            return pageSize;
+        }
+    }
+
     /**
      * @param cancel Cancel flag.
      */

http://git-wip-us.apache.org/repos/asf/ignite/blob/a634e559/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 37e0108..0dd0f6a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -65,6 +65,7 @@ import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEMORY_POLICY_INITIAL_SIZE;
 import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEM_PLC_DEFAULT_NAME;
+import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_PAGE_SIZE;
 
 /**
  *
@@ -350,6 +351,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
      * @param memCfg configuration to validate.
      */
     private void validateConfiguration(MemoryConfiguration memCfg) throws IgniteCheckedException
{
+        checkPageSize(memCfg);
+
         MemoryPolicyConfiguration[] plcCfgs = memCfg.getMemoryPolicies();
 
         Set<String> plcNames = (plcCfgs != null) ? U.<String>newHashSet(plcCfgs.length)
: new HashSet<String>(0);
@@ -381,6 +384,14 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
     }
 
     /**
+     * @param memCfg Memory config.
+     */
+    protected void checkPageSize(MemoryConfiguration memCfg) {
+        if (memCfg.getPageSize() == 0)
+            memCfg.setPageSize(DFLT_PAGE_SIZE);
+    }
+
+    /**
      * @param plcCfg Memory policy config.
      *
      * @throws IgniteCheckedException if validation of memory metrics properties fails.

http://git-wip-us.apache.org/repos/asf/ignite/blob/a634e559/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index e6c5379..6052a7c 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -48,7 +48,7 @@ public class FilePageStore implements PageStore {
     public static final int VERSION = 1;
 
     /** Allocated field offset. */
-    static final int HEADER_SIZE = 8/*SIGNATURE*/ + 4/*VERSION*/ + 1/*type*/ + 4/*page size*/;
+    public static final int HEADER_SIZE = 8/*SIGNATURE*/ + 4/*VERSION*/ + 1/*type*/ + 4/*page
size*/;
 
     /** */
     private final File cfgFile;

http://git-wip-us.apache.org/repos/asf/ignite/blob/a634e559/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 524286c..cbe2cfd 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -10057,6 +10057,47 @@ public abstract class IgniteUtils {
     }
 
     /**
+     * @param path Path.
+     * @param name Name.
+     */
+    public static Path searchFileRecursively(Path path, @NotNull final String name) throws
IgniteCheckedException {
+        final AtomicReference<Path> res = new AtomicReference<>();
+
+        try {
+            Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
+                @Override public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs) {
+                    if (name.equals(file.getFileName().toString())) {
+                        res.set(file);
+
+                        return FileVisitResult.TERMINATE;
+                    }
+
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override public FileVisitResult visitFileFailed(Path file, IOException exc)
{
+                    U.error(null, "file skipped during recursive search - " + file, exc);
+
+                    // Ignoring.
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override public FileVisitResult postVisitDirectory(Path dir, IOException
exc) {
+                    if (exc != null)
+                        U.error(null, "error during recursive search - " + dir, exc);
+
+                    // Ignoring.
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            throw new IgniteCheckedException("walkFileTree will not throw IOException if
the FileVisitor does not");
+        }
+
+        return res.get();
+    }
+
+    /**
      * Returns {@link GridIntIterator} for range of primitive integers.
      * @param start Start.
      * @param cnt Count.

http://git-wip-us.apache.org/repos/asf/ignite/blob/a634e559/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/DefaultPageSizeBackwardsCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/DefaultPageSizeBackwardsCompatibilityTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/DefaultPageSizeBackwardsCompatibilityTest.java
new file mode 100644
index 0000000..e2951e2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/DefaultPageSizeBackwardsCompatibilityTest.java
@@ -0,0 +1,147 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.db.file;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class DefaultPageSizeBackwardsCompatibilityTest extends GridCommonAbstractTest {
+    /** Ip finder. */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** Client mode. */
+    private boolean set2kPageSize = true;
+
+    /** Entries count. */
+    public static final int ENTRIES_COUNT = 300;
+
+    /** Cache name. */
+    public static final String CACHE_NAME = "cache1";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception
{
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpDiscoverySpi discoverySpi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
+        discoverySpi.setIpFinder(IP_FINDER);
+
+        MemoryConfiguration memCfg = new MemoryConfiguration();
+
+        if (set2kPageSize)
+            memCfg.setPageSize(2048);
+
+        MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+        memPlcCfg.setMaxSize(100 * 1000 * 1000);
+
+        memPlcCfg.setName("dfltMemPlc");
+
+        memCfg.setMemoryPolicies(memPlcCfg);
+        memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+        cfg.setMemoryConfiguration(memCfg);
+
+        CacheConfiguration ccfg1 = new CacheConfiguration();
+
+        ccfg1.setName(CACHE_NAME);
+        ccfg1.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+        ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+        ccfg1.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+        cfg.setCacheConfiguration(ccfg1);
+
+        cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration().setCheckpointingFrequency(3_000));
+
+        cfg.setConsistentId(gridName);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        deleteWorkFiles();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        deleteWorkFiles();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartFrom2kDefaultStore() throws Exception {
+        startGrids(2);
+
+        Ignite ig = ignite(0);
+
+        ig.active(true);
+
+        awaitPartitionMapExchange();
+
+        IgniteCache<Integer, Integer> cache = ig.getOrCreateCache(CACHE_NAME);
+
+        for (int i = 0; i < ENTRIES_COUNT; i++)
+            cache.put(i, i);
+
+        Thread.sleep(5_000); // Await for checkpoint to happen.
+
+        stopAllGrids();
+
+        set2kPageSize = false;
+
+        startGrids(2);
+
+        ig = ignite(0);
+
+        ig.active(true);
+
+        awaitPartitionMapExchange();
+
+        cache = ig.getOrCreateCache(CACHE_NAME);
+
+        for (int i = 0; i < ENTRIES_COUNT; i++)
+            assertEquals((Integer)i, cache.get(i));
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    private void deleteWorkFiles() throws IgniteCheckedException {
+        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/a634e559/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index 5762c02..b2a1f65 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -23,6 +23,7 @@ import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsClientNe
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.file.DefaultPageSizeBackwardsCompatibilityTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCheckpointSimulationWithRealCpDisabledTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsEvictionTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreePageMemoryImplTest;
@@ -77,6 +78,8 @@ public class IgnitePdsTestSuite extends TestSuite {
 
         suite.addTestSuite(IgnitePdsCacheRestoreTest.class);
 
+        suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
+
         return suite;
     }
 }


Mime
View raw message