Return-Path: X-Original-To: apmail-ignite-commits-archive@minotaur.apache.org Delivered-To: apmail-ignite-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 76F0018555 for ; Mon, 23 Nov 2015 19:07:14 +0000 (UTC) Received: (qmail 44222 invoked by uid 500); 23 Nov 2015 19:07:13 -0000 Delivered-To: apmail-ignite-commits-archive@ignite.apache.org Received: (qmail 43124 invoked by uid 500); 23 Nov 2015 19:07:11 -0000 Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.apache.org Delivered-To: mailing list commits@ignite.apache.org Received: (qmail 42877 invoked by uid 99); 23 Nov 2015 19:07:10 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 23 Nov 2015 19:07:10 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9763AE0984; Mon, 23 Nov 2015 19:07:10 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sboikov@apache.org To: commits@ignite.apache.org Date: Mon, 23 Nov 2015 19:07:27 -0000 Message-Id: In-Reply-To: <558777b8adeb41a89dbaf1c6379bf097@git.apache.org> References: <558777b8adeb41a89dbaf1c6379bf097@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [18/27] ignite git commit: ignite-yardstick: added debug info into IgniteTransactionalWriteInvokeBenchmark ignite-yardstick: added debug info into IgniteTransactionalWriteInvokeBenchmark Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/fa7a4bce Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/fa7a4bce Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/fa7a4bce Branch: refs/heads/ignite-single-op-get Commit: fa7a4bce65421da4f86c72e78850d987278f8e8f Parents: d71f612 Author: ashutak Authored: Mon Nov 23 16:26:29 2015 +0300 Committer: ashutak Committed: Mon Nov 23 16:26:29 2015 +0300 ---------------------------------------------------------------------- .../IgniteAtomicInvokeRetryBenchmark.java | 2 +- .../failover/IgniteConsistencyException.java | 64 +++++++++ ...IgniteTransactionalInvokeRetryBenchmark.java | 2 +- ...IgniteTransactionalWriteInvokeBenchmark.java | 135 ++++++++++++++----- .../IgniteTransactionalWriteReadBenchmark.java | 2 +- 5 files changed, 171 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/fa7a4bce/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicInvokeRetryBenchmark.java ---------------------------------------------------------------------- diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicInvokeRetryBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicInvokeRetryBenchmark.java index c0567ef..6e65746 100644 --- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicInvokeRetryBenchmark.java +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicInvokeRetryBenchmark.java @@ -110,7 +110,7 @@ public class IgniteAtomicInvokeRetryBenchmark extends IgniteFailoverAbstractBenc } - throw new IllegalStateException("Cache and local map are in inconsistent state " + + throw new IgniteConsistencyException("Cache and local map are in inconsistent state " + "[badKeys=" + badCacheEntries.keySet() + ']'); } http://git-wip-us.apache.org/repos/asf/ignite/blob/fa7a4bce/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteConsistencyException.java ---------------------------------------------------------------------- diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteConsistencyException.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteConsistencyException.java new file mode 100644 index 0000000..e1fcbfa --- /dev/null +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteConsistencyException.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.yardstick.cache.failover; + +import org.jetbrains.annotations.Nullable; + +/** + * Ignite consistency exception. + */ +public class IgniteConsistencyException extends RuntimeException { + /** */ + private static final long serialVersionUID = 0; + + /** + * Create empty exception. + */ + public IgniteConsistencyException() { + // No-op. + } + + /** + * Creates new exception with given error message. + * + * @param msg Error message. + */ + public IgniteConsistencyException(String msg) { + super(msg); + } + + /** + * Creates new grid exception with given throwable as a cause and + * source of error message. + * + * @param cause Non-null throwable cause. + */ + public IgniteConsistencyException(Throwable cause) { + this(cause.getMessage(), cause); + } + + /** + * Creates new exception with given error message and optional nested exception. + * + * @param msg Error message. + * @param cause Optional nested exception (can be {@code null}). + */ + public IgniteConsistencyException(String msg, @Nullable Throwable cause) { + super(msg, cause); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/fa7a4bce/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalInvokeRetryBenchmark.java ---------------------------------------------------------------------- diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalInvokeRetryBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalInvokeRetryBenchmark.java index f8a1689..16b0959 100644 --- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalInvokeRetryBenchmark.java +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalInvokeRetryBenchmark.java @@ -122,7 +122,7 @@ public class IgniteTransactionalInvokeRetryBenchmark extends IgniteFailoverAbstr } } - throw new IllegalStateException("Cache and local map are in inconsistent state."); + throw new IgniteConsistencyException("Cache and local map are in inconsistent state."); } println("Cache validation successfully finished in " http://git-wip-us.apache.org/repos/asf/ignite/blob/fa7a4bce/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteInvokeBenchmark.java ---------------------------------------------------------------------- diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteInvokeBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteInvokeBenchmark.java index 2c4050b..a52ea78 100644 --- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteInvokeBenchmark.java +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteInvokeBenchmark.java @@ -17,14 +17,25 @@ package org.apache.ignite.yardstick.cache.failover; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; import javax.cache.processor.EntryProcessorException; import javax.cache.processor.MutableEntry; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.cache.CacheEntryProcessor; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.IgniteKernal; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition; +import org.apache.ignite.internal.util.typedef.F; +import org.yardstickframework.BenchmarkConfiguration; import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ; @@ -44,12 +55,40 @@ import static org.yardstickframework.BenchmarkUtils.println; * */ public class IgniteTransactionalWriteInvokeBenchmark extends IgniteFailoverAbstractBenchmark { + /** */ + private static final Long INITIAL_VALUE = 1L; + /** {@inheritDoc} */ - @Override public boolean test(Map ctx) throws Exception { - final int k = nextRandom(args.range()); + @Override public void setUp(BenchmarkConfiguration cfg) throws Exception { + super.setUp(cfg); assert args.keysCount() > 0 : "Count of keys: " + args.keysCount(); + println(cfg, "Populating data..."); + + long start = System.nanoTime(); + + if (cfg.memberId() == 0) { + try (IgniteDataStreamer dataLdr = ignite().dataStreamer(cacheName())) { + for (int k = 0; k < args.range() && !Thread.currentThread().isInterrupted(); k++) { + dataLdr.addData("key-" + k + "-master", INITIAL_VALUE); + + for (int i = 0; i < args.keysCount(); i++) + dataLdr.addData("key-" + k + "-" + i, INITIAL_VALUE); + + if (k % 100000 == 0) + println(cfg, "Populated accounts: " + k); + } + } + } + + println(cfg, "Finished populating data in " + ((System.nanoTime() - start) / 1_000_000) + " ms."); + } + + /** {@inheritDoc} */ + @Override public boolean test(Map ctx) throws Exception { + final int k = nextRandom(args.range()); + final String[] keys = new String[args.keysCount()]; final String masterKey = "key-" + k + "-master"; @@ -59,8 +98,10 @@ public class IgniteTransactionalWriteInvokeBenchmark extends IgniteFailoverAbstr final int scenario = nextRandom(2); - return doInTransaction(ignite().transactions(), PESSIMISTIC, REPEATABLE_READ, new Callable() { - @Override public Boolean call() throws Exception { + final Set badKeys = new LinkedHashSet<>(); + + doInTransaction(ignite().transactions(), PESSIMISTIC, REPEATABLE_READ, new Callable() { + @Override public Void call() throws Exception { final int timeout = args.cacheOperationTimeoutMillis(); switch (scenario) { @@ -81,46 +122,40 @@ public class IgniteTransactionalWriteInvokeBenchmark extends IgniteFailoverAbstr Set values = new HashSet<>(map.values()); - if (values.size() != 1) { - // Print all usefull information and finish. - println(cfg, "Got different values for keys [map=" + map + "]"); - - println(cfg, "Cache content:"); - - for (int k = 0; k < args.range(); k++) { - for (int i = 0; i < args.keysCount(); i++) { - String key = "key-" + k + "-" + i; - - asyncCache.get(key); - Long val = asyncCache.future().get(timeout); - - if (val != null) - println(cfg, "Entry [key=" + key + ", val=" + val + "]"); - } - } - - throw new IllegalStateException("Found different values for keys (see above information)."); - } + if (values.size() != 1) + throw new IgniteConsistencyException("Found different values for keys [map="+map+"]"); break; case 1: // Invoke scenario. asyncCache.get(masterKey); Long val = asyncCache.future().get(timeout); - asyncCache.put(masterKey, val == null ? 0 : val + 1); + if (val == null) + badKeys.add(masterKey); + + asyncCache.put(masterKey, val == null ? -1 : val + 1); asyncCache.future().get(timeout); for (String key : keys) { - asyncCache.invoke(key, new IncrementCacheEntryProcessor()); - asyncCache.future().get(timeout); + asyncCache.invoke(key, new IncrementCacheEntryProcessor(), cacheName()); + Object o = asyncCache.future().get(timeout); + + if (o != null) + badKeys.add(key); } break; } - return true; + return null; } }); + + if (!F.isEmpty(badKeys)) + throw new IgniteConsistencyException("Found unexpected null-value(s) for the following " + + "key(s) (look for debug information on server nodes): " + badKeys); + + return true; } /** {@inheritDoc} */ @@ -130,14 +165,52 @@ public class IgniteTransactionalWriteInvokeBenchmark extends IgniteFailoverAbstr /** */ - private static class IncrementCacheEntryProcessor implements CacheEntryProcessor { + private static class IncrementCacheEntryProcessor implements CacheEntryProcessor { /** */ private static final long serialVersionUID = 0; /** {@inheritDoc} */ - @Override public Void process(MutableEntry entry, + @Override public Object process(MutableEntry entry, Object... arguments) throws EntryProcessorException { - entry.setValue(entry.getValue() == null ? 0 : entry.getValue() + 1); + if (entry.getValue() == null) { + String cacheName = (String)arguments[0]; + + IgniteKernal kernal = (IgniteKernal)entry.unwrap(Ignite.class); + + Affinity aff = kernal.affinity(cacheName); + + final int partIdx = aff.partition(entry.getKey()); + + final Collection nodes = aff.mapKeyToPrimaryAndBackups(entry.getKey()); + + List locPartitions = kernal.cachex(cacheName).context().topology(). + localPartitions(); + + GridDhtLocalPartition part = null; + + for (GridDhtLocalPartition p : locPartitions) { + if (p.id() == partIdx) { + part = p; + + break; + } + } + + kernal.log().warning("Found unexpected null-value, debug info:" + + "\n entry=" + entry + + "\n key=" + entry.getKey() + + "\n locNodeId=" + kernal.cluster().localNode().id() + + "\n primaryAndBackupsNodes=" + nodes + + "\n part=" + part + + "\n partIdx=" + partIdx + + "\n locParts=" + locPartitions + + "\n allPartMap=" + kernal.cachex(cacheName).context().topology().partitionMap(true) + ); + + return new Object(); // non-null value. + } + + entry.setValue(entry.getValue() + 1); return null; } http://git-wip-us.apache.org/repos/asf/ignite/blob/fa7a4bce/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteReadBenchmark.java ---------------------------------------------------------------------- diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteReadBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteReadBenchmark.java index f278e8a..c4314ed 100644 --- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteReadBenchmark.java +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteReadBenchmark.java @@ -80,7 +80,7 @@ public class IgniteTransactionalWriteReadBenchmark extends IgniteFailoverAbstrac } } - throw new IllegalStateException("Found different values for keys (see above information)."); + throw new IgniteConsistencyException("Found different values for keys (see above information)."); } final Long oldVal = map.get(keys[0]);