ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dpav...@apache.org
Subject ignite git commit: IGNITE-10045 Add fail-fast mode to bounded iteration of StandaloneWalRecordsIterator - Fixes #5199.
Date Thu, 01 Nov 2018 16:41:55 GMT
Repository: ignite
Updated Branches:
  refs/heads/master 6f9c702cd -> db05c8bb2


IGNITE-10045 Add fail-fast mode to bounded iteration of StandaloneWalRecordsIterator - Fixes
#5199.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/db05c8bb
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/db05c8bb
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/db05c8bb

Branch: refs/heads/master
Commit: db05c8bb244e409603bcae6142e280e55f2c54f1
Parents: 6f9c702
Author: Alexey Stelmak <spiderru5597@gmail.com>
Authored: Thu Nov 1 19:41:38 2018 +0300
Committer: Dmitriy Pavlov <dpavlov@apache.org>
Committed: Thu Nov 1 19:41:38 2018 +0300

----------------------------------------------------------------------
 .../wal/reader/IgniteWalIteratorFactory.java    |  16 ++-
 .../reader/StandaloneWalRecordsIterator.java    |  56 ++++++++-
 .../StandaloneWalRecordsIteratorTest.java       | 115 ++++++++++++++++++-
 3 files changed, 181 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/db05c8bb/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
index 4ea1828..f4ebec5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
@@ -181,7 +181,8 @@ public class IgniteWalIteratorFactory {
             iteratorParametersBuilder.lowBound,
             iteratorParametersBuilder.highBound,
             iteratorParametersBuilder.keepBinary,
-            iteratorParametersBuilder.bufferSize
+            iteratorParametersBuilder.bufferSize,
+            iteratorParametersBuilder.strictBoundsCheck
         );
     }
 
@@ -418,6 +419,9 @@ public class IgniteWalIteratorFactory {
         /** */
         private FileWALPointer highBound = DFLT_HIGH_BOUND;
 
+        /** Use strict bounds check for WAL segments. */
+        private boolean strictBoundsCheck;
+
         /**
          * @param filesOrDirs Paths to files or directories.
          * @return IteratorParametersBuilder Self reference.
@@ -535,6 +539,16 @@ public class IgniteWalIteratorFactory {
         }
 
         /**
+         * @param flag Use strict check.
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder strictBoundsCheck(boolean flag) {
+            this.strictBoundsCheck = flag;
+
+            return this;
+        }
+
+        /**
          * Copy current state of builder to new instance.
          *
          * @return IteratorParametersBuilder Self reference.

http://git-wip-us.apache.org/repos/asf/ignite/blob/db05c8bb/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index 25432d3..be5f55a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -22,6 +22,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.GridKernalContext;
@@ -116,7 +117,8 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator
{
         FileWALPointer lowBound,
         FileWALPointer highBound,
         boolean keepBinary,
-        int initialReadBufferSize
+        int initialReadBufferSize,
+        boolean strictBoundsCheck
     ) throws IgniteCheckedException {
         super(
             log,
@@ -127,6 +129,9 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator
{
             FILE_INPUT_FACTORY
         );
 
+        if (strictBoundsCheck)
+            strictCheck(walFiles, lowBound, highBound);
+
         this.lowBound = lowBound;
         this.highBound = highBound;
 
@@ -140,6 +145,55 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator
{
     }
 
     /**
+     * @param walFiles Wal files.
+     * @return printable indexes of segment files.
+     */
+    private static String printIndexes(List<FileDescriptor> walFiles) {
+        return "[" + String.join(",", walFiles.stream().map(f -> Long.toString(f.idx())).collect(Collectors.toList()))
+ "]";
+    }
+
+    /**
+     * @param walFiles Wal files.
+     * @param lowBound Low bound.
+     * @param highBound High bound.
+     *
+     * @throws IgniteCheckedException if failed
+     */
+    private static void strictCheck(List<FileDescriptor> walFiles, FileWALPointer lowBound,
FileWALPointer highBound) throws IgniteCheckedException {
+        int idx = 0;
+
+        if (lowBound.index() > Long.MIN_VALUE) {
+            for (; idx < walFiles.size(); idx++) {
+                FileDescriptor desc = walFiles.get(idx);
+
+                assert desc != null;
+
+                if (desc.idx() == lowBound.index())
+                    break;
+            }
+        }
+
+        if (idx == walFiles.size())
+            throw new IgniteCheckedException("Wal segments not in bounds. loBoundIndex="
+ lowBound.index() +
+                                                ", indexes=" + printIndexes(walFiles));
+
+        long curWalSegmIdx = walFiles.get(idx).idx();
+
+        for (; idx < walFiles.size() && curWalSegmIdx <= highBound.index();
idx++, curWalSegmIdx++) {
+            FileDescriptor desc = walFiles.get(idx);
+
+            assert desc != null;
+
+            if (curWalSegmIdx != desc.idx())
+                throw new IgniteCheckedException("Wal segment " + curWalSegmIdx + " not found
in files " + printIndexes(walFiles));
+        }
+
+        if (highBound.index() < Long.MAX_VALUE && curWalSegmIdx <= highBound.index())
+            throw new IgniteCheckedException("Wal segments not in bounds. hiBoundIndex="
+ highBound.index() +
+                                                ", indexes=" + printIndexes(walFiles));
+    }
+
+    /**
      * For directory mode sets oldest file as initial segment, for file by file mode, converts
all files to descriptors
      * and gets oldest as initial.
      *

http://git-wip-us.apache.org/repos/asf/ignite/blob/db05c8bb/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
index 4d2bdcf..aa24479 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
@@ -21,6 +21,8 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.OpenOption;
 import java.nio.file.StandardOpenOption;
+import java.util.List;
+import java.util.Random;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
@@ -30,17 +32,23 @@ import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.RolloverType;
 import org.apache.ignite.internal.pagemem.wal.record.SnapshotRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder;
@@ -89,11 +97,9 @@ public class StandaloneWalRecordsIteratorTest extends GridCommonAbstractTest
{
     }
 
     /**
-     * Check correct closing file descriptors.
      *
-     * @throws Exception if test failed.
      */
-    public void testCorrectClosingFileDescriptors() throws Exception {
+    private String createWalFiles() throws Exception {
         IgniteEx ig = (IgniteEx)startGrid();
 
         String archiveWalDir = getArchiveWalDirPath(ig);
@@ -118,8 +124,18 @@ public class StandaloneWalRecordsIteratorTest extends GridCommonAbstractTest
{
 
         stopGrid();
 
+        return archiveWalDir;
+    }
+
+    /**
+     * Check correct closing file descriptors.
+     *
+     * @throws Exception if test failed.
+     */
+    public void testCorrectClosingFileDescriptors() throws Exception {
+
         // Iterate by all archived WAL segments.
-        createWalIterator(archiveWalDir).forEach(x -> {
+        createWalIterator(createWalFiles()).forEach(x -> {
         });
 
         assertTrue("At least one WAL file must be opened!", CountedFileIO.getCountOpenedWalFiles()
> 0);
@@ -128,6 +144,62 @@ public class StandaloneWalRecordsIteratorTest extends GridCommonAbstractTest
{
     }
 
     /**
+     * Check correct check bounds.
+     *
+     * @throws Exception if test failed.
+     */
+    public void testStrictBounds() throws Exception {
+        String dir = createWalFiles();
+
+        FileWALPointer lowBound = null, highBound = null;
+
+        for (IgniteBiTuple<WALPointer, WALRecord> p : createWalIterator(dir, null,
null, false)) {
+            if (lowBound == null)
+                lowBound = (FileWALPointer) p.get1();
+
+            highBound = (FileWALPointer) p.get1();
+        }
+
+        assertNotNull(lowBound);
+
+        assertNotNull(highBound);
+
+        createWalIterator(dir, lowBound, highBound, true);
+
+        final FileWALPointer lBound = lowBound;
+        final FileWALPointer hBound = highBound;
+
+        //noinspection ThrowableNotThrown
+        GridTestUtils.assertThrows(log, () -> {
+            createWalIterator(dir, new FileWALPointer(lBound.index() - 1, 0, 0), hBound,
true);
+
+            return 0;
+        } , IgniteCheckedException.class, null);
+
+        //noinspection ThrowableNotThrown
+        GridTestUtils.assertThrows(log, () -> {
+            createWalIterator(dir, lBound, new FileWALPointer(hBound.index() + 1, 0, 0),
true);
+
+            return 0;
+        }, IgniteCheckedException.class, null);
+
+        List<FileDescriptor> walFiles = listWalFiles(dir);
+
+        assertNotNull(walFiles);
+
+        assertTrue(!walFiles.isEmpty());
+
+        assertTrue(walFiles.get(new Random().nextInt(walFiles.size())).file().delete());
+
+        //noinspection ThrowableNotThrown
+        GridTestUtils.assertThrows(log, () -> {
+            createWalIterator(dir, lBound, hBound, true);
+
+            return 0;
+        }, IgniteCheckedException.class, null);
+    }
+
+    /**
      * Creates WALIterator associated with files inside walDir.
      *
      * @param walDir - path to WAL directory.
@@ -142,6 +214,41 @@ public class StandaloneWalRecordsIteratorTest extends GridCommonAbstractTest
{
         return new IgniteWalIteratorFactory(log).iterator(params.filesOrDirs(walDir));
     }
 
+
+    /**
+     * @param walDir Wal directory.
+     */
+    private List<FileDescriptor> listWalFiles(String walDir) throws IgniteCheckedException
{
+        IteratorParametersBuilder params = new IteratorParametersBuilder();
+
+        params.ioFactory(new RandomAccessFileIOFactory());
+
+        return new IgniteWalIteratorFactory(log).resolveWalFiles(params.filesOrDirs(walDir));
+    }
+
+    /**
+     * @param walDir Wal directory.
+     * @param lowBound Low bound.
+     * @param highBound High bound.
+     * @param strictCheck Strict check.
+     */
+    private WALIterator createWalIterator(String walDir, FileWALPointer lowBound, FileWALPointer
highBound, boolean strictCheck)
+                    throws IgniteCheckedException {
+        IteratorParametersBuilder params = new IteratorParametersBuilder();
+
+        params.ioFactory(new RandomAccessFileIOFactory()).
+            filesOrDirs(walDir).
+            strictBoundsCheck(strictCheck);
+
+        if (lowBound != null)
+            params.from(lowBound);
+
+        if (lowBound != null)
+            params.to(highBound);
+
+        return new IgniteWalIteratorFactory(log).iterator(params);
+    }
+
     /**
      * Evaluate path to directory with WAL archive.
      *


Mime
View raw message