ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From agoncha...@apache.org
Subject [16/19] ignite git commit: ignite-6788 Ignite WAL reader fails on Tx marker record for persistent store with new style folder naming
Date Wed, 01 Nov 2017 09:55:46 GMT
ignite-6788 Ignite WAL reader fails on Tx marker record for persistent store with new style
folder naming

Signed-off-by: agura <agura@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5542c70d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5542c70d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5542c70d

Branch: refs/heads/ignite-6778
Commit: 5542c70da913ff1006973e25f5112152674880db
Parents: 67ca745
Author: dpavlov <dpavlov@gridgain.com>
Authored: Wed Nov 1 03:17:00 2017 +0300
Committer: agura <agura@apache.org>
Committed: Wed Nov 1 03:17:00 2017 +0300

----------------------------------------------------------------------
 .../wal/reader/StandaloneGridKernalContext.java |  20 +-
 .../db/wal/reader/IgniteWalReaderTest.java      | 283 ++++++++++++-------
 2 files changed, 195 insertions(+), 108 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/5542c70d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index c0c3650..485458b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.internal.processors.cache.persistence.wal.reader;
 
 import java.io.File;
-import java.io.Serializable;
 import java.lang.reflect.Field;
 import java.util.Iterator;
 import java.util.List;
@@ -49,8 +48,8 @@ import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager;
 import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
 import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
 import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
-import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.closure.GridClosureProcessor;
 import org.apache.ignite.internal.processors.cluster.ClusterProcessor;
@@ -95,6 +94,9 @@ import org.jetbrains.annotations.Nullable;
  * Dummy grid kernal context
  */
 public class StandaloneGridKernalContext implements GridKernalContext {
+    /** Binary metadata file store folderŅŽ */
+    public static final String BINARY_META_FOLDER = "binary_meta";
+
     /** Config for fake Ignite instance. */
     private final IgniteConfiguration cfg;
 
@@ -123,13 +125,12 @@ public class StandaloneGridKernalContext implements GridKernalContext
{
      * Providing {@code null} will disable unmarshall for non primitive objects, BinaryObjects
will be provided <br>
      */
     StandaloneGridKernalContext(IgniteLogger log,
-        @Nullable final File binaryMetadataFileStoreDir,
-        @Nullable final File marshallerMappingFileStoreDir) throws IgniteCheckedException
{
+        @Nullable File binaryMetadataFileStoreDir,
+        @Nullable File marshallerMappingFileStoreDir) throws IgniteCheckedException {
         this.log = log;
 
         try {
-            pluginProc = new StandaloneIgnitePluginProcessor(
-                this, config());
+            pluginProc = new StandaloneIgnitePluginProcessor(this, config());
         }
         catch (IgniteCheckedException e) {
             throw new IllegalStateException("Must not fail on empty providers list.", e);
@@ -137,7 +138,12 @@ public class StandaloneGridKernalContext implements GridKernalContext
{
 
         this.marshallerCtx = new MarshallerContextImpl(null);
         this.cfg = prepareIgniteConfiguration();
-        this.cacheObjProcessor = binaryMetadataFileStoreDir != null ? binaryProcessor(this,
binaryMetadataFileStoreDir) : null;
+
+        // Fake folder provided to perform processor startup on empty folder.
+        if (binaryMetadataFileStoreDir == null)
+            binaryMetadataFileStoreDir = new File(BINARY_META_FOLDER).getAbsoluteFile();
+
+        this.cacheObjProcessor = binaryProcessor(this, binaryMetadataFileStoreDir);
 
         if (marshallerMappingFileStoreDir != null) {
             marshallerCtx.setMarshallerMappingFileStoreDir(marshallerMappingFileStoreDir);

http://git-wip-us.apache.org/repos/asf/ignite/blob/5542c70d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
index 88eacef..9348a68 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
@@ -31,6 +31,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -40,6 +41,7 @@ import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteEvents;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
@@ -68,8 +70,12 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.reader.Ignite
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.logger.NullLogger;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.transactions.Transaction;
 import org.jetbrains.annotations.NotNull;
@@ -77,6 +83,7 @@ import org.jetbrains.annotations.Nullable;
 import org.junit.Assert;
 
 import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.CREATE;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
 import static org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor.genNewStyleSubfolderName;
@@ -94,12 +101,6 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
     /** additional cache for testing different combinations of types in WAL */
     private static final String CACHE_ADDL_NAME = "cache1";
 
-    /** Delete DB dir before test. */
-    private static final boolean deleteBefore = true;
-
-    /** Delete DB dir after test. */
-    private static final boolean deleteAfter = true;
-
     /** Dump records to logger. Should be false for non local run */
     private static final boolean dumpRecords = false;
 
@@ -115,6 +116,9 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
     /** Custom wal mode. */
     private WALMode customWalMode;
 
+    /** Clear properties in afterTest method() */
+    private boolean clearProperties;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception
{
         final IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -151,23 +155,22 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
     @Override protected void beforeTest() throws Exception {
         stopAllGrids();
 
-        if (deleteBefore)
-            deleteWorkFiles();
+        deleteWorkFiles();
     }
 
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         stopAllGrids();
 
-        if (deleteAfter)
-            deleteWorkFiles();
+        if (clearProperties)
+            System.clearProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS);
     }
 
     /**
      * @throws IgniteCheckedException If failed.
      */
-    private void deleteWorkFiles() throws IgniteCheckedException {
-        deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR,
false));
+    private void deleteWorkFiles() throws Exception {
+        GridTestUtils.deleteDbFiles();
     }
 
     /**
@@ -197,17 +200,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         final int cntUsingMockIter = iterateAndCount(it, false);
 
         log.info("Total records loaded " + cntUsingMockIter);
-        assert cntUsingMockIter > 0;
-        assert cntUsingMockIter > cacheObjectsToWrite;
+        assertTrue(cntUsingMockIter > 0);
+        assertTrue(cntUsingMockIter > cacheObjectsToWrite);
 
         final File walArchiveDirWithConsistentId = new File(walArchive, subfolderName);
         final File walWorkDirWithConsistentId = new File(wal, subfolderName);
-
-        final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
-        final File binaryMetaWithConsId = new File(binaryMeta, subfolderName);
-        final File marshaller = U.resolveWorkDirectory(workDir, "marshaller", false);
-
-        final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, PAGE_SIZE,
binaryMetaWithConsId, marshaller);
+        final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
         final int cntArchiveDir = iterateAndCount(factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId));
 
         log.info("Total records loaded using directory : " + cntArchiveDir);
@@ -218,12 +216,13 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
 
         log.info("Total records loaded using archive directory (file-by-file): " + cntArchiveFileByFile);
 
-        assert cntArchiveFileByFile > cacheObjectsToWrite;
-        assert cntArchiveDir > cacheObjectsToWrite;
-        assert cntArchiveDir == cntArchiveFileByFile;
+        assertTrue(cntArchiveFileByFile > cacheObjectsToWrite);
+        assertTrue(cntArchiveDir > cacheObjectsToWrite);
+        assertTrue(cntArchiveDir == cntArchiveFileByFile);
         //really count2 may be less because work dir correct loading is not supported yet
-        assert cntUsingMockIter >= cntArchiveDir
-            : "Mock based reader loaded " + cntUsingMockIter + " records but standalone has
loaded only " + cntArchiveDir;
+        assertTrue("Mock based reader loaded " + cntUsingMockIter + " records " +
+                "but standalone has loaded only " + cntArchiveDir,
+            cntUsingMockIter >= cntArchiveDir);
 
         final File[] workFiles = walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
 
@@ -231,10 +230,10 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
 
         log.info("Total records loaded from work: " + cntWork);
 
-        assert cntWork + cntArchiveFileByFile == cntUsingMockIter
-            : "Work iterator loaded [" + cntWork + "] " +
-            "Archive iterator loaded [" + cntArchiveFileByFile + "]; " +
-            "mock iterator [" + cntUsingMockIter + "]";
+        assertTrue("Work iterator loaded [" + cntWork + "] " +
+                "Archive iterator loaded [" + cntArchiveFileByFile + "]; " +
+                "mock iterator [" + cntUsingMockIter + "]",
+            cntWork + cntArchiveFileByFile == cntUsingMockIter);
     }
 
     /**
@@ -312,7 +311,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         putDummyRecords(ignite, 500);
 
         stopGrid("node0");
-        assert evtRecorded.get();
+        assertTrue(evtRecorded.get());
     }
 
     /**
@@ -407,7 +406,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
             archiveSegmentForInactivity.await(archiveIncompleteSegmentAfterInactivityMs +
1001, TimeUnit.MILLISECONDS);
 
         stopGrid("node0");
-        assert recordedAfterSleep;
+        assertTrue(recordedAfterSleep);
     }
 
     /**
@@ -460,8 +459,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
             binaryMetaWithConsId,
             marshallerMapping);
 
-        final BiConsumer<Object, Object> objConsumer = new BiConsumer<Object, Object>()
{
-            @Override public void accept(Object key, Object val) {
+        final IgniteBiInClosure<Object, Object> objConsumer = new IgniteBiInClosure<Object,
Object>() {
+            @Override public void apply(Object key, Object val) {
                 boolean rmv = remove(ctrlMap, key, val);
                 if (!rmv)
                     log.error("Unable to remove Key and value from control Map K:[" + key
+ "] V: [" + val + "]");
@@ -471,14 +470,14 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
                     assertEquals(indexedObj.iVal, indexedObj.jVal);
                     assertEquals(indexedObj.iVal, key);
                     for (byte datum : indexedObj.getData()) {
-                        assert datum >= 'A' && datum <= 'A' + 10;
+                        assertTrue(datum >= 'A' && datum <= 'A' + 10);
                     }
                 }
             }
         };
         scanIterateAndCount(factory, workDir, subfolderName, cntEntries, txCnt, objConsumer,
null);
 
-        assert ctrlMap.isEmpty() : " Control Map is not empty after reading entries " + ctrlMap;
+        assertTrue(" Control Map is not empty after reading entries " + ctrlMap, ctrlMap.isEmpty());
     }
 
     /**
@@ -510,8 +509,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         final String subfolderName,
         final int minCntEntries,
         final int minTxCnt,
-        @Nullable final BiConsumer<Object, Object> objConsumer,
-        @Nullable final Consumer<DataRecord> dataRecordHnd) throws IgniteCheckedException
{
+        @Nullable final IgniteBiInClosure<Object, Object> objConsumer,
+        @Nullable final IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException
{
 
         final File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false);
         final File wal = new File(db, "wal");
@@ -544,8 +543,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         final int entriesWork = valuesSum(cntWork.values());
         log.info("Archive directory: Tx found " + txCntObservedWork + " entries " + entriesWork);
 
-        assert entriesArch + entriesWork >= minCntEntries;
-        assert txCntObservedWork + txCntObservedArch >= minTxCnt;
+        assertTrue("entriesArch=" + entriesArch + " + entriesWork=" + entriesWork
+                + " >= minCntEntries=" + minCntEntries,
+            entriesArch + entriesWork >= minCntEntries);
+        assertTrue("txCntObservedWork=" + txCntObservedWork + " + txCntObservedArch=" + txCntObservedArch
+                + " >= minTxCnt=" + minTxCnt,
+            txCntObservedWork + txCntObservedArch >= minTxCnt);
     }
 
     /**
@@ -611,11 +614,9 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         final File binaryMetaWithNodeSubfolder = new File(binaryMeta, subfolderName);
         final File marshallerMapping = U.resolveWorkDirectory(workDir, "marshaller", false);
 
-        final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, PAGE_SIZE,
-            binaryMetaWithNodeSubfolder,
-            marshallerMapping);
-        final BiConsumer<Object, Object> objConsumer = new BiConsumer<Object, Object>()
{
-            @Override public void accept(Object key, Object val) {
+        final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
+        final IgniteBiInClosure<Object, Object> objConsumer = new IgniteBiInClosure<Object,
Object>() {
+            @Override public void apply(Object key, Object val) {
                 log.info("K: [" + key + ", " +
                     (key != null ? key.getClass().getName() : "?") + "]" +
                     " V: [" + val + ", " +
@@ -625,12 +626,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
                     String msg = "Unable to remove pair from control map " + "K: [" + key
+ "] V: [" + val + "]";
                     log.error(msg);
                 }
-                assert !(val instanceof BinaryObject);
+                assertFalse(val instanceof BinaryObject);
             }
         };
 
-        final Consumer<DataRecord> toStrChecker = new Consumer<DataRecord>()
{
-            @Override public void accept(DataRecord record) {
+        final IgniteInClosure<DataRecord> toStrChecker = new IgniteInClosure<DataRecord>()
{
+            @Override public void apply(DataRecord record) {
                 String strRepresentation = record.toString();
                 for (Iterator<String> iter = ctrlStringsToSearch.iterator(); iter.hasNext();
) {
                     final String next = iter.next();
@@ -643,17 +644,17 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         };
         scanIterateAndCount(factory, workDir, subfolderName, cntEntries, 0, objConsumer,
toStrChecker);
 
-        assert ctrlMap.isEmpty() : " Control Map is not empty after reading entries: " +
ctrlMap;
-        assert ctrlStringsToSearch.isEmpty() : " Control Map for strings in entries is not
empty after" +
-            " reading records: " + ctrlStringsToSearch;
+        assertTrue(" Control Map is not empty after reading entries: " + ctrlMap, ctrlMap.isEmpty());
+        assertTrue(" Control Map for strings in entries is not empty after" +
+            " reading records: " + ctrlStringsToSearch, ctrlStringsToSearch.isEmpty());
 
         //Validate same WAL log with flag binary objects only
         final IgniteWalIteratorFactory keepBinFactory = new IgniteWalIteratorFactory(log,
PAGE_SIZE,
             binaryMetaWithNodeSubfolder,
             marshallerMapping,
             true);
-        final BiConsumer<Object, Object> binObjConsumer = new BiConsumer<Object,
Object>() {
-            @Override public void accept(Object key, Object val) {
+        final IgniteBiInClosure<Object, Object> binObjConsumer = new IgniteBiInClosure<Object,
Object>() {
+            @Override public void apply(Object key, Object val) {
                 log.info("K(KeepBinary): [" + key + ", " +
                     (key != null ? key.getClass().getName() : "?") + "]" +
                     " V(KeepBinary): [" + val + ", " +
@@ -693,15 +694,15 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
 
                         byte data[] = binaryObj.field("data");
                         for (byte datum : data) {
-                            assert datum >= 'A' && datum <= 'A' + 10;
+                            assertTrue(datum >= 'A' && datum <= 'A' + 10);
                         }
                     }
                 }
             }
         };
 
-        final Consumer<DataRecord> binObjToStrChecker = new Consumer<DataRecord>()
{
-            @Override public void accept(DataRecord record) {
+        final IgniteInClosure<DataRecord> binObjToStrChecker = new IgniteInClosure<DataRecord>()
{
+            @Override public void apply(DataRecord record) {
                 String strRepresentation = record.toString();
                 for (Iterator<String> iter = ctrlStringsForBinaryObjSearch.iterator();
iter.hasNext(); ) {
                     final String next = iter.next();
@@ -714,9 +715,11 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         };
         scanIterateAndCount(keepBinFactory, workDir, subfolderName, cntEntries, 0, binObjConsumer,
binObjToStrChecker);
 
-        assert ctrlMapForBinaryObjects.isEmpty() : " Control Map is not empty after reading
entries: " + ctrlMapForBinaryObjects;
-        assert ctrlStringsForBinaryObjSearch.isEmpty() : " Control Map for strings in entries
is not empty after" +
-            " reading records: " + ctrlStringsForBinaryObjSearch;
+        assertTrue(" Control Map is not empty after reading entries: " + ctrlMapForBinaryObjects,
+            ctrlMapForBinaryObjects.isEmpty());
+        assertTrue(" Control Map for strings in entries is not empty after" +
+                " reading records: " + ctrlStringsForBinaryObjSearch,
+            ctrlStringsForBinaryObjSearch.isEmpty());
 
     }
 
@@ -764,7 +767,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         stopGrid("node0");
 
         final String workDir = U.defaultWorkDirectory();
-        final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName,
workDir);
+        final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
 
         scanIterateAndCount(factory, workDir, subfolderName, totalEntries, 0, null, null);
     }
@@ -787,7 +790,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         stopGrid("node0");
 
         final String workDir = U.defaultWorkDirectory();
-        final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName,
workDir);
+        final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
 
         scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, null);
     }
@@ -859,13 +862,13 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         stopGrid("node0");
 
         final String workDir = U.defaultWorkDirectory();
-        final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName,
workDir);
+        final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
 
         final StringBuilder builder = new StringBuilder();
         final Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
 
-        scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, new Consumer<DataRecord>()
{
-            @Override public void accept(DataRecord dataRecord) {
+        scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, new IgniteInClosure<DataRecord>()
{
+            @Override public void apply(DataRecord dataRecord) {
                 final List<DataEntry> entries = dataRecord.writeEntries();
 
                 builder.append("{");
@@ -902,14 +905,121 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
     }
 
     /**
-     * @param subfolderName Subfolder name.
+     * Tests transaction generation and WAL for putAll cache operation.
+     * @throws Exception if failed.
+     */
+    public void testPutAllTxIntoTwoNodes() throws Exception {
+        final Ignite ignite = startGrid("node0");
+        final Ignite ignite1 = startGrid(1);
+
+        ignite.active(true);
+
+        final Map<Object, IndexedObject> map = new TreeMap<>();
+
+        final int cntEntries = 1000;
+        for (int i = 0; i < cntEntries; i++)
+            map.put(i, new IndexedObject(i));
+
+        ignite.cache(CACHE_NAME).putAll(map);
+
+        ignite.active(false);
+
+        final String subfolderName = genDbSubfolderName(ignite, 0);
+        final String subfolderName1 = genDbSubfolderName(ignite1, 1);
+
+        stopAllGrids();
+
+        final String workDir = U.defaultWorkDirectory();
+        final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
+
+        final StringBuilder builder = new StringBuilder();
+        final Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
+
+        final IgniteInClosure<DataRecord> drHnd = new IgniteInClosure<DataRecord>()
{
+            @Override public void apply(DataRecord dataRecord) {
+                final List<DataEntry> entries = dataRecord.writeEntries();
+
+                builder.append("{");
+                for (DataEntry entry : entries) {
+                    final GridCacheOperation op = entry.op();
+                    final Integer cnt = operationsFound.get(op);
+
+                    operationsFound.put(op, cnt == null ? 1 : (cnt + 1));
+
+                    if (entry instanceof UnwrapDataEntry) {
+                        final UnwrapDataEntry entry1 = (UnwrapDataEntry)entry;
+
+                        builder.append(entry1.op()).append(" for ").append(entry1.unwrappedKey());
+                        final GridCacheVersion ver = entry.nearXidVersion();
+
+                        builder.append(", ");
+
+                        if (ver != null)
+                            builder.append("tx=").append(ver).append(", ");
+                    }
+                }
+
+                builder.append("}\n");
+            }
+        };
+        scanIterateAndCount(factory, workDir, subfolderName, 1, 1, null, drHnd);
+        scanIterateAndCount(factory, workDir, subfolderName1, 1, 1, null, drHnd);
+
+        final Integer createsFound = operationsFound.get(CREATE);
+
+        if (log.isInfoEnabled())
+            log.info(builder.toString());
+
+        assertTrue("Create operations should be found in log: " + operationsFound,
+            createsFound != null && createsFound > 0);
+
+        assertTrue("Create operations count should be at least " + cntEntries + " in log:
" + operationsFound,
+            createsFound != null && createsFound >= cntEntries);
+
+    }
+
+    /**
+     * Tests transaction generation and WAL for putAll cache operation.
+     * @throws Exception if failed.
+     */
+    public void testTxRecordsReadWoBinaryMeta() throws Exception {
+        clearProperties = true;
+        System.setProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS, "true");
+
+        final Ignite ignite = startGrid("node0");
+        ignite.active(true);
+
+        final Map<Object, IndexedObject> map = new TreeMap<>();
+
+        for (int i = 0; i < 1000; i++)
+            map.put(i, new IndexedObject(i));
+
+        ignite.cache(CACHE_NAME).putAll(map);
+
+        ignite.active(false);
+
+        final String workDir = U.defaultWorkDirectory();
+        final String subfolderName = genDbSubfolderName(ignite, 0);
+        stopAllGrids();
+
+        IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger(),
+            PAGE_SIZE,
+            null,
+            null,
+            false);
+
+        scanIterateAndCount(factory, workDir, subfolderName, 1000, 1, null, null);
+    }
+
+    /**
      * @param workDir Work directory.
+     * @param subfolderName Subfolder name.
      * @return WAL iterator factory.
      * @throws IgniteCheckedException If failed.
      */
     @NotNull private IgniteWalIteratorFactory createWalIteratorFactory(
-        String subfolderName,
-        String workDir
+        final String workDir,
+        final String subfolderName
     ) throws IgniteCheckedException {
         final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
         final File binaryMetaWithConsId = new File(binaryMeta, subfolderName);
@@ -918,7 +1028,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         return new IgniteWalIteratorFactory(log,
             PAGE_SIZE,
             binaryMetaWithConsId,
-            marshallerMapping);
+            marshallerMapping,
+            false);
     }
 
     /**
@@ -943,8 +1054,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
      */
     private Map<GridCacheVersion, Integer> iterateAndCountDataRecord(
         final WALIterator walIter,
-        @Nullable final BiConsumer<Object, Object> cacheObjHnd,
-        @Nullable final Consumer<DataRecord> dataRecordHnd) throws IgniteCheckedException
{
+        @Nullable final IgniteBiInClosure<Object, Object> cacheObjHnd,
+        @Nullable final IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException
{
 
         final Map<GridCacheVersion, Integer> entriesUnderTxFound = new HashMap<>();
 
@@ -957,7 +1068,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
                     final DataRecord dataRecord = (DataRecord)walRecord;
 
                     if (dataRecordHnd != null)
-                        dataRecordHnd.accept(dataRecord);
+                        dataRecordHnd.apply(dataRecord);
                     final List<DataEntry> entries = dataRecord.writeEntries();
 
                     for (DataEntry entry : entries) {
@@ -990,7 +1101,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
                             "; Value: " + unwrappedValObj);
 
                         if (cacheObjHnd != null && (unwrappedKeyObj != null || unwrappedValObj
!= null))
-                            cacheObjHnd.accept(unwrappedKeyObj, unwrappedValObj);
+                            cacheObjHnd.apply(unwrappedKeyObj, unwrappedValObj);
 
                         final Integer entriesUnderTx = entriesUnderTxFound.get(globalTxId);
                         entriesUnderTxFound.put(globalTxId, entriesUnderTx == null ? 1 :
entriesUnderTx + 1);
@@ -1008,36 +1119,6 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         return entriesUnderTxFound;
     }
 
-    /**
-     * Represents an operation that accepts a single input argument and returns no
-     * result.
-     *
-     * @param <T>
-     */
-    private interface Consumer<T> {
-        /**
-         * Performs this operation on the given argument.
-         *
-         * @param t the input argument
-         */
-        public void accept(T t);
-    }
-
-    /**
-     * Represents an operation that accepts two input arguments and returns no
-     * result.
-     *
-     * @param <T>
-     */
-    private interface BiConsumer<T, U> {
-        /**
-         * Performs this operation on the given argument.
-         *
-         * @param t the input argument
-         */
-        public void accept(T t, U u);
-    }
-
     /** Enum for cover binaryObject enum save/load */
     enum TestEnum {
         /** */A, /** */B, /** */C


Mime
View raw message