ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From agoncha...@apache.org
Subject [1/2] ignite git commit: IGNITE-6277 Added ability to convert WAL to human readable form - Fixes #2605.
Date Fri, 15 Sep 2017 10:48:56 GMT
Repository: ignite
Updated Branches:
  refs/heads/master 91cf66e70 -> 319aa8402


http://git-wip-us.apache.org/repos/asf/ignite/blob/319aa840/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index cd0f8ab..5bbd7da 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -92,7 +92,7 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
         @NotNull FileIOFactory ioFactory) throws IgniteCheckedException {
         super(log,
             sharedCtx,
-            new RecordV1Serializer(sharedCtx),
+            new RecordV1Serializer(sharedCtx, true),
             ioFactory,
             BUF_SIZE);
         init(walFilesDir, false, null);

http://git-wip-us.apache.org/repos/asf/ignite/blob/319aa840/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/record/HeaderRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/record/HeaderRecord.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/record/HeaderRecord.java
index 3fc5637..4fa6232 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/record/HeaderRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/record/HeaderRecord.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.cache.persistence.wal.record;
 
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
  * Header record.
@@ -47,4 +48,9 @@ public class HeaderRecord extends WALRecord {
     @Override public RecordType type() {
         return RecordType.HEADER_RECORD;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(HeaderRecord.class, this, "super", super.toString());
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/319aa840/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
index 663fe0c..91e1f00 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
@@ -136,11 +136,22 @@ public class RecordV1Serializer implements RecordSerializer {
     /** Skip CRC calculation/check flag */
     private boolean skipCrc = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CRC, false);
 
+    /** Write pointer. */
+    private final boolean writePointer;
+
     /**
      * @param cctx Cache shared context.
      */
     public RecordV1Serializer(GridCacheSharedContext cctx) {
+        this(cctx, false);
+    }
+
+    /**
+     * @param cctx Cache shared context.
+     */
+    public RecordV1Serializer(GridCacheSharedContext cctx, boolean writePointer) {
         this.cctx = cctx;
+        this.writePointer = writePointer;
 
         co = cctx.kernalContext().cacheObjects();
         pageSize = cctx.database().pageSize();
@@ -674,7 +685,15 @@ public class RecordV1Serializer implements RecordSerializer {
 
             assert res != null;
 
-            res.size((int)(in0.position() - startPos + CRC_SIZE)); // Account for CRC which
will be read afterwards.
+            int size = (int) (in0.position() - startPos + CRC_SIZE);
+
+            res.size(size); // Account for CRC which will be read afterwards.
+
+            if (writePointer && expPtr instanceof FileWALPointer) {
+                FileWALPointer ptr = (FileWALPointer) expPtr;
+
+                res.position(new FileWALPointer(ptr.index(), ptr.fileOffset(), size));
+            }
 
             return res;
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/319aa840/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
index d50b8e8..16fcb45 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -165,12 +165,6 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest
{
     @Override protected void beforeTest() throws Exception {
         deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
 
-//        long seed = System.currentTimeMillis();
-//
-//        info("Seed: " + seed + "L");
-//
-//        BPlusTree.rnd = new Random(seed);
-
         startGrids(gridCount());
 
         if (withClientNearCache()) {

http://git-wip-us.apache.org/repos/asf/ignite/blob/319aa840/modules/dev-utils/pom.xml
----------------------------------------------------------------------
diff --git a/modules/dev-utils/pom.xml b/modules/dev-utils/pom.xml
new file mode 100644
index 0000000..2ab0683
--- /dev/null
+++ b/modules/dev-utils/pom.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    POM file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.ignite</groupId>
+        <artifactId>ignite-parent</artifactId>
+        <version>1</version>
+        <relativePath>../../parent</relativePath>
+    </parent>
+
+    <artifactId>ignite-dev-utils</artifactId>
+    <version>2.2.0-SNAPSHOT</version>
+    <url>http://ignite.apache.org</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-indexing</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+</project>

http://git-wip-us.apache.org/repos/asf/ignite/blob/319aa840/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
----------------------------------------------------------------------
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
new file mode 100644
index 0000000..d0f0a2b
--- /dev/null
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.development.utils;
+
+import java.io.File;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
+import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasInnerIO;
+import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasLeafIO;
+import org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO;
+import org.apache.ignite.internal.processors.query.h2.database.io.H2LeafIO;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.logger.NullLogger;
+
+/**
+ * Print WAL log data in human-readable form.
+ */
+public class IgniteWalConverter {
+    /**
+     * @param args Args.
+     */
+    public static void main(String[] args) throws Exception {
+        if (args.length < 2)
+            throw new IllegalArgumentException("\nYou need to provide:\n" +
+                    "\t1. Size of pages (1024, 2048, etc).\n" +
+                    "\t2. Path to dir with wal files.\n" +
+                    "\t3. (Optional) Path to dir with archive wal files.");
+
+        PageIO.registerH2(H2InnerIO.VERSIONS, H2LeafIO.VERSIONS);
+        H2ExtrasInnerIO.register();
+        H2ExtrasLeafIO.register();
+
+        final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger(),
+                new RandomAccessFileIOFactory(), new Integer(args[0]));
+
+        final File walWorkDirWithConsistentId = new File(args[1]);
+
+        final File[] workFiles = walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
+
+        if (workFiles == null)
+            throw new IllegalArgumentException("No .wal files in dir: " + args[1]);
+
+        try (WALIterator stIt = factory.iteratorWorkFiles(workFiles)) {
+            while (stIt.hasNextX()) {
+                IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
+
+                System.out.println("[W] " + next.get2());
+            }
+        }
+
+        if (args.length >= 3) {
+            final File walArchiveDirWithConsistentId = new File(args[2]);
+
+            try (WALIterator stIt = factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId))
{
+                while (stIt.hasNextX()) {
+                    IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
+
+                    System.out.println("[A] " + next.get2());
+                }
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/319aa840/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 49048db..cfa266d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,6 +51,7 @@
         <module>modules/apache-license-gen</module>
         <module>modules/tools</module>
         <module>modules/core</module>
+        <module>modules/dev-utils</module>
         <module>modules/hadoop</module>
         <module>modules/extdata/p2p</module>
         <module>modules/extdata/uri</module>


Mime
View raw message