Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id F3952200D27 for ; Wed, 25 Oct 2017 11:58:59 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id F2134160C08; Wed, 25 Oct 2017 09:58:59 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id EDBEB160BF8 for ; Wed, 25 Oct 2017 11:58:58 +0200 (CEST) Received: (qmail 9268 invoked by uid 500); 25 Oct 2017 09:58:57 -0000 Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.apache.org Delivered-To: mailing list commits@ignite.apache.org Received: (qmail 8897 invoked by uid 99); 25 Oct 2017 09:58:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Oct 2017 09:58:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 5EF62E115D; Wed, 25 Oct 2017 09:58:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sboikov@apache.org To: commits@ignite.apache.org Date: Wed, 25 Oct 2017 09:59:18 -0000 Message-Id: <9f28f5a04d8d4a1ba79efdc1a94e8f33@git.apache.org> In-Reply-To: <97e04c24a4334e98ad803e8907490b62@git.apache.org> References: <97e04c24a4334e98ad803e8907490b62@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [26/50] [abbrv] ignite git commit: IGNITE-6577 Check temporary file existence outside lock file during cache start - Fixes #2817. archived-at: Wed, 25 Oct 2017 09:59:00 -0000 IGNITE-6577 Check temporary file existence outside lock file during cache start - Fixes #2817. Signed-off-by: Alexey Goncharuk Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7a61c15c Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7a61c15c Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7a61c15c Branch: refs/heads/ignite-5937 Commit: 7a61c15c746bd77f4bbce05c0db509c467fec68a Parents: bb31a8a Author: dpavlov Authored: Mon Oct 23 17:08:07 2017 +0300 Committer: Alexey Goncharuk Committed: Mon Oct 23 17:08:07 2017 +0300 ---------------------------------------------------------------------- .../persistence/file/FilePageStoreManager.java | 62 ++++++++++++-------- .../snapshot/IgniteCacheSnapshotManager.java | 3 + .../wal/serializer/RecordV2Serializer.java | 3 +- .../ignite/internal/util/IgniteUtils.java | 62 +++++++++++--------- 4 files changed, 76 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/7a61c15c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index aadcee6..1fe22ca 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -28,6 +28,7 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -389,43 +390,52 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen File lockF = new File(cacheWorkDir, IgniteCacheSnapshotManager.SNAPSHOT_RESTORE_STARTED_LOCK_FILENAME); - if (lockF.exists()) { - Path tmp = cacheWorkDir.toPath().getParent().resolve(cacheWorkDir.getName() + ".tmp"); + Path cacheWorkDirPath = cacheWorkDir.toPath(); - boolean deleted = U.delete(cacheWorkDir); + Path tmp = cacheWorkDirPath.getParent().resolve(cacheWorkDir.getName() + ".tmp"); - if (Files.exists(tmp) && Files.isDirectory(tmp)) { - U.warn(log, "Ignite node crashed during the snapshot restore process " + - "(there is a snapshot restore lock file left for cache). But old version of cache was saved. " + - "Trying to restore it. Cache - [" + cacheWorkDir.getAbsolutePath() + ']'); + if (Files.exists(tmp) && Files.isDirectory(tmp) && + Files.exists(tmp.resolve(IgniteCacheSnapshotManager.TEMP_FILES_COMPLETENESS_MARKER))) { - try { - Files.move(tmp, cacheWorkDir.toPath()); - } - catch (IOException e) { - throw new IgniteCheckedException(e); - } - } - else { - U.warn(log, "Ignite node crashed during the snapshot restore process " + - "(there is a snapshot restore lock file left for cache). Will remove both the lock file and " + - "incomplete cache directory [cacheDir=" + cacheWorkDir.getAbsolutePath() + ']'); + U.warn(log, "Ignite node crashed during the snapshot restore process " + + "(there is a snapshot restore lock file left for cache). But old version of cache was saved. " + + "Trying to restore it. Cache - [" + cacheWorkDir.getAbsolutePath() + ']'); - if (!deleted) - throw new IgniteCheckedException("Failed to remove obsolete cache working directory " + - "(remove the directory manually and make sure the work folder has correct permissions): " + - cacheWorkDir.getAbsolutePath()); + U.delete(cacheWorkDir); - cacheWorkDir.mkdirs(); + try { + Files.move(tmp, cacheWorkDirPath, StandardCopyOption.ATOMIC_MOVE); + + cacheWorkDirPath.resolve(IgniteCacheSnapshotManager.TEMP_FILES_COMPLETENESS_MARKER).toFile().delete(); + } + catch (IOException e) { + throw new IgniteCheckedException(e); } + } + else if (lockF.exists()) { + U.warn(log, "Ignite node crashed during the snapshot restore process " + + "(there is a snapshot restore lock file left for cache). Will remove both the lock file and " + + "incomplete cache directory [cacheDir=" + cacheWorkDir.getAbsolutePath() + ']'); - if (!cacheWorkDir.exists()) - throw new IgniteCheckedException("Failed to initialize cache working directory " + - "(failed to create, make sure the work folder has correct permissions): " + + boolean deleted = U.delete(cacheWorkDir); + + if (!deleted) + throw new IgniteCheckedException("Failed to remove obsolete cache working directory " + + "(remove the directory manually and make sure the work folder has correct permissions): " + cacheWorkDir.getAbsolutePath()); + + cacheWorkDir.mkdirs(); } else dirExisted = true; + + if (!cacheWorkDir.exists()) + throw new IgniteCheckedException("Failed to initialize cache working directory " + + "(failed to create, make sure the work folder has correct permissions): " + + cacheWorkDir.getAbsolutePath()); + + if (Files.exists(tmp)) + U.delete(tmp); } return dirExisted; http://git-wip-us.apache.org/repos/asf/ignite/blob/7a61c15c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java index 50e6515..5746c17 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java @@ -39,6 +39,9 @@ public class IgniteCacheSnapshotManager extends Gri /** Snapshot started lock filename. */ public static final String SNAPSHOT_RESTORE_STARTED_LOCK_FILENAME = "snapshot-started.loc"; + /** Temp files completeness marker. */ + public static final String TEMP_FILES_COMPLETENESS_MARKER = "finished.tmp"; + /** * Try to start local snapshot operation if it's required by discovery event. * http://git-wip-us.apache.org/repos/asf/ignite/blob/7a61c15c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java index 98804d9..5eb45ac 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java @@ -32,6 +32,7 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTai import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.io.RecordIO; import org.apache.ignite.internal.util.typedef.F; +import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.*; import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.CRC_SIZE; import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.REC_TYPE_SIZE; @@ -39,7 +40,7 @@ import static org.apache.ignite.internal.processors.cache.persistence.wal.serial * Record V2 serializer. * Stores records in following format: *
    - *
  • Record type from {@code WALRecord.RecordType} incremented by 1
    • + *
  • Record type from {@link RecordType#ordinal()} incremented by 1
    • *
  • WAL pointer to double check consistency
    • *
  • Record length
    • *
  • Data
    • http://git-wip-us.apache.org/repos/asf/ignite/blob/7a61c15c/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java index bdcf87e..d426468 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java @@ -3434,7 +3434,6 @@ public abstract class IgniteUtils { return e; } - /** * Deletes file or directory with all sub-directories and files. * @@ -3443,40 +3442,49 @@ public abstract class IgniteUtils { * {@code false} otherwise */ public static boolean delete(@Nullable File file) { - if (file == null) - return false; - - boolean res = true; - - if (file.isDirectory()) { - File[] files = file.listFiles(); + return file != null && delete(file.toPath()); + } - if (files != null && files.length > 0) { - for (File file1 : files) { - if (file1.isDirectory()) - res &= delete(file1); - else if (file1.getName().endsWith("jar")) { - try { - // Why do we do this? - new JarFile(file1, false).close(); - } - catch (IOException ignore) { - // Ignore it here... - } + /** + * Deletes file or directory with all sub-directories and files. + * + * @param path File or directory to delete. + * @return {@code true} if and only if the file or directory is successfully deleted, + * {@code false} otherwise + */ + public static boolean delete(Path path) { + if (Files.isDirectory(path)) { + try { + try (DirectoryStream stream = Files.newDirectoryStream(path)) { + for (Path innerPath : stream) { + boolean res = delete(innerPath); - res &= file1.delete(); + if (!res) + return false; } - else - res &= file1.delete(); } + } catch (IOException e) { + return false; } + } - res &= file.delete(); + if (path.endsWith("jar")) { + try { + // Why do we do this? + new JarFile(path.toString(), false).close(); + } + catch (IOException ignore) { + // Ignore it here... + } } - else - res = file.delete(); - return res; + try { + Files.delete(path); + + return true; + } catch (IOException e) { + return false; + } } /**