ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dpav...@apache.org
Subject [ignite] 02/03: IGNITE-11030 Fix Pages List to preserve partition ID consistency. (#5915)
Date Mon, 25 Mar 2019 15:31:22 GMT
This is an automated email from the ASF dual-hosted git repository.

dpavlov pushed a commit to branch ignite-2.7.5
in repository https://gitbox.apache.org/repos/asf/ignite.git

commit df73704f409c511b6acc3dbace2acd3a2f4a683b
Author: Dmitriy Pavlov <dpavlov@apache.org>
AuthorDate: Thu Jan 31 18:45:47 2019 +0300

    IGNITE-11030 Fix Pages List to preserve partition ID consistency. (#5915)
    
    (cherry picked from commit ec488539933f1cafa0756c7d2bd22b48f0c1e16b)
---
 .../wal/record/delta/InitNewPageRecord.java        | 10 +++
 .../cache/persistence/freelist/PagesList.java      |  8 ++-
 .../cache/persistence/pagemem/PageMemoryImpl.java  | 16 ++++-
 .../persistence/db/IgniteTcBotInitNewPageTest.java | 81 ++++++++++++++++++++++
 .../testsuites/IgnitePdsWithIndexingTestSuite.java |  2 +
 5 files changed, 113 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
index d0ba2aa..9ed7f35 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.pagemem.wal.record.delta;
 
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -51,6 +52,15 @@ public class InitNewPageRecord extends PageDeltaRecord {
         this.ioType = ioType;
         this.ioVer = ioVer;
         this.newPageId = newPageId;
+
+        int newPartId = PageIdUtils.partId(newPageId);
+        int partId = PageIdUtils.partId(pageId);
+
+        if (newPartId != partId) {
+            throw new AssertionError("Partition consistency failure: " +
+                "newPageId=" + Long.toHexString(newPageId) + " (newPartId: " + newPartId
+ ") " +
+                "pageId=" + Long.toHexString(pageId) + " (partId: " + partId + ")");
+        }
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
index f1cc32a..9a7bee9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
@@ -1145,9 +1145,11 @@ public abstract class PagesList extends DataStructure {
 
                         decrementBucketSize(bucket);
 
-                        if (initIoVers != null)
-                            dataPageId = initReusedPage(tailId, tailPage, tailAddr, 0, FLAG_DATA,
initIoVers.latest());
-                        else
+                        if (initIoVers != null) {
+                            int partId = PageIdUtils.partId(tailId);
+
+                            dataPageId = initReusedPage(tailId, tailPage, tailAddr, partId,
FLAG_DATA, initIoVers.latest());
+                        } else
                             dataPageId = recyclePage(tailId, tailPage, tailAddr, null);
 
                         dirty = true;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 5c91378..6e00a10 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -1569,7 +1569,7 @@ public class PageMemoryImpl implements PageMemoryEx {
         long pageId = PageIO.getPageId(page + PAGE_OVERHEAD);
 
         assert pageId != 0 : U.hexLong(PageHeader.readPageId(page));
-        assert PageIO.getVersion(page + PAGE_OVERHEAD) != 0 : U.hexLong(pageId);
+        assert PageIO.getVersion(page + PAGE_OVERHEAD) != 0 : dumpPage(pageId, fullId.groupId());
         assert PageIO.getType(page + PAGE_OVERHEAD) != 0 : U.hexLong(pageId);
 
         try {
@@ -1586,6 +1586,20 @@ public class PageMemoryImpl implements PageMemoryEx {
     }
 
     /**
+     * Prepares page details for assertion.
+     * @param pageId Page id.
+     * @param grpId Group id.
+     */
+    @NotNull private String dumpPage(long pageId, int grpId) {
+        int pageIdx = PageIdUtils.pageIndex(pageId);
+        int partId = PageIdUtils.partId(pageId);
+        long off = (long)(pageIdx + 1) * pageSize();
+
+        return U.hexLong(pageId) + " (grpId=" + grpId + ", pageIdx=" + pageIdx + ", partId="
+ partId + ", offH=" +
+            Long.toHexString(off) + ")";
+    }
+
+    /**
      * @param absPtr Absolute pointer to the page.
      * @return {@code True} if write lock acquired for the page.
      */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteTcBotInitNewPageTest.java
b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteTcBotInitNewPageTest.java
new file mode 100644
index 0000000..8ef517c
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteTcBotInitNewPageTest.java
@@ -0,0 +1,81 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import com.google.common.base.Strings;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+/**
+ * Test creates a lot of index pages in the cache with low number of partitions.<br>
+ * Then cache entries are removed to enforce all pages to come to a free list. <br>
+ * Then creation of data pages with long data will probably result in page rotation.<br>
+ * Expected behaviour: all {@link InitNewPageRecord} should have consistent partition IDs.
+ */
+public class IgniteTcBotInitNewPageTest extends GridCommonAbstractTest {
+    /** Cache name. */
+    public static final String CACHE = "cache";
+
+    /** */
+    @Test
+    public void testInitNewPagePageIdConsistency() throws Exception {
+        IgniteEx ex = startGrid(0);
+
+        IgniteCache<Object, Object> cache = ex.cache(CACHE);
+
+        for (int i = 0; i < 1_000_000; i++)
+            cache.put(i, i);
+
+        cache.clear();
+
+        for (int i = 0; i < 1_000; i++)
+            cache.put(i, Strings.repeat("Apache Ignite", 1000));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws
Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        CacheConfiguration<Integer, Object> ccfg = new CacheConfiguration<>(CACHE);
+
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 4));
+
+        cfg.setCacheConfiguration(ccfg);
+
+        DataRegionConfiguration regCfg = new DataRegionConfiguration()
+            .setMaxSize(2L * 1024 * 1024 * 1024)
+            .setPersistenceEnabled(true);
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+            .setWalMode(WALMode.LOG_ONLY)
+            .setDefaultDataRegionConfiguration(regCfg);
+
+        cfg.setDataStorageConfiguration(dsCfg);
+
+        return cfg;
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
index 67b9fad..88e3fe7 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
@@ -20,6 +20,7 @@ package org.apache.ignite.testsuites;
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest;
 import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgniteTcBotInitNewPageTest;
 import org.apache.ignite.internal.processors.database.IgniteDbMultiNodeWithIndexingPutGetTest;
 import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingPutGetTest;
 import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingWalRestoreTest;
@@ -46,6 +47,7 @@ public class IgnitePdsWithIndexingTestSuite extends TestSuite {
         suite.addTestSuite(IgnitePersistentStoreSchemaLoadTest.class);
         suite.addTestSuite(IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest.class);
         suite.addTestSuite(IgniteTwoRegionsRebuildIndexTest.class);
+        suite.addTestSuite(IgniteTcBotInitNewPageTest.class);
 
         return suite;
     }


Mime
View raw message