Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id E8486200D1A for ; Mon, 9 Oct 2017 12:42:19 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id DE48E160BEC; Mon, 9 Oct 2017 10:42:19 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 11DBF1609BB for ; Mon, 9 Oct 2017 12:42:18 +0200 (CEST) Received: (qmail 9375 invoked by uid 500); 9 Oct 2017 10:42:18 -0000 Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.apache.org Delivered-To: mailing list commits@ignite.apache.org Received: (qmail 9365 invoked by uid 99); 9 Oct 2017 10:42:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 09 Oct 2017 10:42:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7837FF5D46; Mon, 9 Oct 2017 10:42:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sboikov@apache.org To: commits@ignite.apache.org Date: Mon, 09 Oct 2017 10:42:17 -0000 Message-Id: <2ee6a66401e14202b9b282582a273719@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [01/25] ignite git commit: IGNITE-6554 Atomic cache remove operations are not logged to WAL - Fixes #2800. archived-at: Mon, 09 Oct 2017 10:42:20 -0000 Repository: ignite Updated Branches: refs/heads/ignite-3478 a1d9ddd10 -> 921404a6f IGNITE-6554 Atomic cache remove operations are not logged to WAL - Fixes #2800. Signed-off-by: Alexey Goncharuk Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/20033c73 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/20033c73 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/20033c73 Branch: refs/heads/ignite-3478 Commit: 20033c7324c49212ad9543e13e9a5ac1e4bf5b68 Parents: 62f3c4c Author: dpavlov Authored: Wed Oct 4 15:05:48 2017 +0300 Committer: Alexey Goncharuk Committed: Wed Oct 4 15:05:48 2017 +0300 ---------------------------------------------------------------------- .../processors/cache/GridCacheMapEntry.java | 2 + .../db/wal/reader/IgniteWalReaderTest.java | 159 ++++++++++++++++++- 2 files changed, 158 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/20033c73/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 7b60b9c..54b8dc3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -4486,6 +4486,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme if (updateCntr != null) updateCntr0 = updateCntr; + entry.logUpdate(op, null, newVer, 0, updateCntr0); + if (oldVal != null) { assert !entry.deletedUnlocked(); http://git-wip-us.apache.org/repos/asf/ignite/blob/20033c73/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java index b9c60b2..6db2784 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java @@ -24,6 +24,7 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import java.io.Serializable; import java.util.Collection; +import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -61,6 +62,7 @@ import org.apache.ignite.internal.pagemem.wal.record.TxRecord; import org.apache.ignite.internal.pagemem.wal.record.UnwrapDataEntry; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.CacheObject; +import org.apache.ignite.internal.processors.cache.GridCacheOperation; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory; @@ -75,8 +77,9 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED; -import static org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor.genNewStyleSubfolderName; +import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR; +import static org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor.genNewStyleSubfolderName; /** * Test suite for WAL segments reader and event generator. @@ -109,6 +112,9 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { */ private int archiveIncompleteSegmentAfterInactivityMs; + /** Custom wal mode. */ + private WALMode customWalMode; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { final IgniteConfiguration cfg = super.getConfiguration(gridName); @@ -143,7 +149,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { pCfg.setWalHistorySize(1); pCfg.setWalSegmentSize(1024 * 1024); pCfg.setWalSegments(WAL_SEGMENTS); - pCfg.setWalMode(WALMode.BACKGROUND); + pCfg.setWalMode(customWalMode != null ? customWalMode : WALMode.BACKGROUND); if (archiveIncompleteSegmentAfterInactivityMs > 0) pCfg.setWalAutoArchiveAfterInactivity(archiveIncompleteSegmentAfterInactivityMs); @@ -709,6 +715,128 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { } /** + * Creates and fills cache with data. + * + * @param ig Ignite instance. + * @param mode Cache Atomicity Mode. + */ + private void createCache2(Ignite ig, CacheAtomicityMode mode) { + if (log.isInfoEnabled()) + log.info("Populating the cache..."); + + final CacheConfiguration cfg = new CacheConfiguration<>("Org" + "11"); + cfg.setAtomicityMode(mode); + final IgniteCache cache = ig.getOrCreateCache(cfg).withKeepBinary(); + + try (Transaction tx = ig.transactions().txStart()) { + for (int i = 0; i < 10; i++) { + + cache.put(i, new Organization(i, "Organization-" + i)); + + if (i % 2 == 0) + cache.put(i, new Organization(i, "Organization-updated-" + i)); + + if (i % 5 == 0) + cache.remove(i); + } + tx.commit(); + } + + } + + /** + * Test if DELETE operation can be found for transactional cache after mixed cache operations including remove(). + * + * @throws Exception if failed. + */ + public void testRemoveOperationPresentedForDataEntry() throws Exception { + runRemoveOperationTest(CacheAtomicityMode.TRANSACTIONAL); + } + + /** + * Test if DELETE operation can be found for atomic cache after mixed cache operations including remove(). + * + * @throws Exception if failed. + */ + public void testRemoveOperationPresentedForDataEntryForAtomic() throws Exception { + runRemoveOperationTest(CacheAtomicityMode.ATOMIC); + } + + + /** + * Test if DELETE operation can be found after mixed cache operations including remove(). + * + * @throws Exception if failed. + * @param mode Cache Atomicity Mode. + */ + private void runRemoveOperationTest(CacheAtomicityMode mode) throws Exception { + final Ignite ignite = startGrid("node0"); + + ignite.active(true); + createCache2(ignite, mode); + ignite.active(false); + + final String subfolderName = genDbSubfolderName(ignite, 0); + + stopGrid("node0"); + + final String workDir = U.defaultWorkDirectory(); + final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName, workDir); + + final StringBuilder builder = new StringBuilder(); + final Map operationsFound = new EnumMap<>(GridCacheOperation.class); + + scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, new Consumer() { + @Override public void accept(DataRecord dataRecord) { + final List entries = dataRecord.writeEntries(); + + builder.append("{"); + for (DataEntry entry : entries) { + final GridCacheOperation op = entry.op(); + final Integer cnt = operationsFound.get(op); + + operationsFound.put(op, cnt == null ? 1 : (cnt + 1)); + + if (entry instanceof UnwrapDataEntry) { + final UnwrapDataEntry entry1 = (UnwrapDataEntry)entry; + + builder.append(entry1.op()).append(" for ").append(entry1.unwrappedKey()); + final GridCacheVersion ver = entry.nearXidVersion(); + + builder.append(", "); + + if (ver != null) + builder.append("tx=").append(ver).append(", "); + } + } + + builder.append("}\n"); + } + }); + + final Integer deletesFound = operationsFound.get(DELETE); + + if (log.isInfoEnabled()) + log.info(builder.toString()); + + assertTrue("Delete operations should be found in log: " + operationsFound, + deletesFound != null && deletesFound > 0); + } + + @NotNull private IgniteWalIteratorFactory createWalIteratorFactory(String subfolderName, + String workDir) throws IgniteCheckedException { + final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false); + final File binaryMetaWithConsId = new File(binaryMeta, subfolderName); + final File marshallerMapping = U.resolveWorkDirectory(workDir, "marshaller", false); + + return new IgniteWalIteratorFactory(log, + PAGE_SIZE, + binaryMetaWithConsId, + marshallerMapping); + } + + + /** * @param values collection with numbers * @return sum of numbers */ @@ -776,7 +904,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { "; Key: " + unwrappedKeyObj + "; Value: " + unwrappedValObj); - if (cacheObjHnd != null && unwrappedKeyObj != null || unwrappedValObj != null) + if (cacheObjHnd != null && (unwrappedKeyObj != null || unwrappedValObj != null)) cacheObjHnd.accept(unwrappedKeyObj, unwrappedValObj); final Integer entriesUnderTx = entriesUnderTxFound.get(globalTxId); @@ -967,4 +1095,29 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { '}'; } } + + /** Test class for storing in ignite */ + private static class Organization { + /** Key. */ + private final int key; + /** Name. */ + private final String name; + + /** + * @param key Key. + * @param name Name. + */ + public Organization(int key, String name) { + this.key = key; + this.name = name; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return "Organization{" + + "key=" + key + + ", name='" + name + '\'' + + '}'; + } + } }