ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From voze...@apache.org
Subject [12/38] ignite git commit: ignite-1397: Load/consistency tests.
Date Thu, 05 Nov 2015 12:17:56 GMT
ignite-1397: Load/consistency tests.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5b0a18dd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5b0a18dd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5b0a18dd

Branch: refs/heads/ignite-1282
Commit: 5b0a18dde8a060428fb9616f0ee39f85cfc34297
Parents: 28e0217
Author: ashutak <ashutak@gridgain.com>
Authored: Mon Nov 2 17:14:04 2015 +0300
Committer: ashutak <ashutak@gridgain.com>
Committed: Mon Nov 2 17:14:04 2015 +0300

----------------------------------------------------------------------
 .../config/benchmark-client-mode.properties     |   4 +-
 .../config/benchmark-failover.properties        | 107 +++++++
 .../config/benchmark-multicast.properties       |  71 ++--
 .../yardstick/config/benchmark-store.properties |   4 +-
 modules/yardstick/config/benchmark.properties   |   4 +-
 .../config/ignite-failover-base-config.xml      | 126 ++++++++
 .../config/ignite-failover-localhost-config.xml |  56 ++++
 modules/yardstick/pom.xml                       |   3 +-
 .../yardstick/IgniteAbstractBenchmark.java      |   2 +-
 .../yardstick/IgniteBenchmarkArguments.java     |  66 +++-
 .../cache/IgniteCacheAbstractBenchmark.java     |   8 +-
 .../yardstick/cache/IgniteGetBenchmark.java     |   4 +-
 .../cache/IgniteJdbcSqlQueryBenchmark.java      |   4 +-
 .../yardstick/cache/IgnitePutAllBenchmark.java  |   4 +-
 .../cache/IgnitePutAllTxBenchmark.java          |   4 +-
 .../yardstick/cache/IgnitePutBenchmark.java     |   4 +-
 .../yardstick/cache/IgnitePutGetBenchmark.java  |   4 +-
 .../cache/IgnitePutGetTxBenchmark.java          |   4 +-
 .../cache/IgnitePutIndexedValue1Benchmark.java  |   4 +-
 .../cache/IgnitePutIndexedValue2Benchmark.java  |   4 +-
 .../cache/IgnitePutIndexedValue8Benchmark.java  |   4 +-
 .../yardstick/cache/IgnitePutTxBenchmark.java   |   4 +-
 .../cache/IgniteSqlQueryBenchmark.java          |   4 +-
 .../cache/IgniteSqlQueryJoinBenchmark.java      |   4 +-
 .../cache/IgniteSqlQueryPutBenchmark.java       |   4 +-
 .../IgniteAtomicInvokeRetryBenchmark.java       | 214 +++++++++++++
 ...IgniteAtomicOffHeapInvokeRetryBenchmark.java |  31 ++
 .../IgniteAtomicOffHeapRetriesBenchmark.java    |  31 ++
 .../failover/IgniteAtomicRetriesBenchmark.java  |  89 ++++++
 .../IgniteFailoverAbstractBenchmark.java        | 320 +++++++++++++++++++
 .../cache/failover/IgniteFailoverNode.java      |  60 ++++
 ...IgniteTransactionalInvokeRetryBenchmark.java | 212 ++++++++++++
 ...ransactionalOffHeapInvokeRetryBenchmark.java |  33 ++
 ...ransactionalOffHeapWriteInvokeBenchmark.java |  37 +++
 ...eTransactionalOffHeapWriteReadBenchmark.java |  32 ++
 ...IgniteTransactionalWriteInvokeBenchmark.java | 182 +++++++++++
 .../IgniteTransactionalWriteReadBenchmark.java  | 141 ++++++++
 37 files changed, 1821 insertions(+), 68 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/config/benchmark-client-mode.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-client-mode.properties b/modules/yardstick/config/benchmark-client-mode.properties
index 8941417..ba5525f 100644
--- a/modules/yardstick/config/benchmark-client-mode.properties
+++ b/modules/yardstick/config/benchmark-client-mode.properties
@@ -17,12 +17,14 @@
 # Contains all multicast benchmarks
 #
 
+now0=`date +'%H%M%S'`
+
 # JVM options.
 JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
--Xloggc:./gc.log \
+-Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/config/benchmark-failover.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-failover.properties b/modules/yardstick/config/benchmark-failover.properties
new file mode 100644
index 0000000..7ed464c
--- /dev/null
+++ b/modules/yardstick/config/benchmark-failover.properties
@@ -0,0 +1,107 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Contains failover benchmarks.
+#
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+#JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false -Xms15g -Xmx15g"
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false -ea"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+  -Xloggc:./gc${now0}.log \
+  -XX:+PrintGCDetails \
+  -verbose:gc \
+  -XX:+UseParNewGC \
+  -XX:+UseConcMarkSweepGC \
+  -XX:+UseTLAB \
+  -XX:NewSize=128m \
+  -XX:MaxNewSize=128m \
+  -XX:MaxTenuringThreshold=0 \
+  -XX:SurvivorRatio=1024 \
+  -XX:+UseCMSInitiatingOccupancyOnly \
+  -XX:CMSInitiatingOccupancyFraction=60 \
+"
+
+#Ignite version
+ver="RELEASE-"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
+BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe,DStatProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+RESTART_SERVERS=true
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# Comma-separated list of the hosts to run BenchmarkServers on.
+SERVER_HOSTS=localhost,localhost
+
+# Comma-separated list of the hosts to run BenchmarkDrivers on.
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Space-separated list of Benchmark driver names (required)
+dn="\
+  IgniteAtomicRetriesBenchmark \
+  IgniteAtomicInvokeRetryBenchmark \
+  IgniteTransactionalWriteReadBenchmark \
+  IgniteTransactionalWriteInvokeBenchmark \
+  IgniteTransactionalInvokeRetryBenchmark \
+  IgniteAtomicOffHeapRetriesBenchmark \
+  IgniteAtomicOffHeapInvokeRetryBenchmark \
+  IgniteTransactionalOffHeapWriteReadBenchmark \
+  IgniteTransactionalOffHeapWriteInvokeBenchmark \
+  IgniteTransactionalOffHeapInvokeRetryBenchmark \
+"
+
+# Warmup 1 min.
+w=60
+
+# Duration 2h = 2*60*60 = 7200 sec.
+d=7200
+
+# Restart delay 10 min = 10 * 60 sec = 600 sec.
+rd=600
+
+# Restart sleep 2 min = 120 sec.
+rs=120
+
+# Keys rang.
+r=100000
+
+# Thread count.
+t=128
+
+# Run configuration which contains all benchmarks.
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-failover-localhost-config.xml -nn ${nodesNum} -b 1 -w ${w} -d ${d} -rd ${rd} -rs ${rs} -r ${r} -t ${t} --client -sm PRIMARY_SYNC -dn ${dn} -sn IgniteFailoverNode -ds failover-1b,\
+-cfg ${SCRIPT_DIR}/../config/ignite-failover-localhost-config.xml -nn ${nodesNum} -b 2 -w ${w} -d ${d} -rd ${rd} -rs ${rs} -r ${r} -t ${t} --client -sm PRIMARY_SYNC -dn ${dn} -sn IgniteFailoverNode -ds failover-2b,\
+"

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/config/benchmark-multicast.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-multicast.properties b/modules/yardstick/config/benchmark-multicast.properties
index 82fc3f8..536ccf4 100644
--- a/modules/yardstick/config/benchmark-multicast.properties
+++ b/modules/yardstick/config/benchmark-multicast.properties
@@ -17,12 +17,14 @@
 # Contains all multicast benchmarks
 #
 
+now0=`date +'%H%M%S'`
+
 # JVM options.
 JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
--Xloggc:./gc.log \
+-Xloggc:./gc${now0}.log \
 -XX:+PrintGCDetails \
 -verbose:gc \
 -XX:+UseParNewGC \
@@ -35,6 +37,7 @@ JVM_OPTS=${JVM_OPTS}" \
 -XX:+UseCMSInitiatingOccupancyOnly \
 -XX:CMSInitiatingOccupancyFraction=60 \
 "
+
 #Ignite version
 ver="RELEASE-"
 
@@ -63,31 +66,49 @@ DRIVER_HOSTS=localhost
 # Number of nodes, used to wait for the specified number of nodes to start.
 nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
 
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
+# Jobs.
+j=10
+
 # Run configuration which contains all benchmarks.
 # Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
 CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxBenchmark -sn IgniteNode -ds ${ver}tx-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryJoinBenchmark -sn IgniteNode -ds ${ver}sql-query-join-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds ${ver}affcall-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteApplyBenchmark -sn IgniteNode -ds ${ver}apply-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteBroadcastBenchmark -sn IgniteNode -ds ${ver}broad-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteExecuteBenchmark -sn IgniteNode -ds ${ver}exec-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteRunBenchmark -sn IgniteNode -ds ${ver}run-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxOffHeapBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryJoinOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-join-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryPutOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -bs 100 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -bs 100 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -bs 100 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-1-backup\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxBenchmark -sn IgniteNode -ds ${ver}tx-put-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-put-get-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryJoinBenchmark -sn IgniteNode -ds ${ver}sql-query-join-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds ${ver}affcall-compute-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteApplyBenchmark -sn IgniteNode -ds ${ver}apply-compute-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteBroadcastBenchmark -sn IgniteNode -ds ${ver}broad-compute-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteExecuteBenchmark -sn IgniteNode -ds ${ver}exec-compute-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteRunBenchmark -sn IgniteNode -ds ${ver}run-compute-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-val-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-val-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxOffHeapBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-val-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-offheap-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryJoinOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-join-offheap-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-put-offheap-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 100 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 100 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 100 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-1-backup\
 "

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/config/benchmark-store.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-store.properties b/modules/yardstick/config/benchmark-store.properties
index cb9e507..bda136e 100644
--- a/modules/yardstick/config/benchmark-store.properties
+++ b/modules/yardstick/config/benchmark-store.properties
@@ -21,12 +21,14 @@
 # - TRANSACTIONAL cache
 #
 
+now0=`date +'%H%M%S'`
+
 # JVM options.
 JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
-  -Xloggc:./gc.log \
+  -Xloggc:./gc${now0}.log \
   -XX:+PrintGCDetails \
   -verbose:gc \
   -XX:+UseParNewGC \

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/config/benchmark.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark.properties b/modules/yardstick/config/benchmark.properties
index d691f1e..67ef5ef 100644
--- a/modules/yardstick/config/benchmark.properties
+++ b/modules/yardstick/config/benchmark.properties
@@ -17,12 +17,14 @@
 # Contains all benchmarks
 #
 
+now0=`date +'%H%M%S'`
+
 # JVM options.
 JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
 
 # Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
 JVM_OPTS=${JVM_OPTS}" \
-  -Xloggc:./gc.log \
+  -Xloggc:./gc${now0}.log \
   -XX:+PrintGCDetails \
   -verbose:gc \
   -XX:+UseParNewGC \

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/config/ignite-failover-base-config.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-failover-base-config.xml b/modules/yardstick/config/ignite-failover-base-config.xml
new file mode 100644
index 0000000..1e1dcff
--- /dev/null
+++ b/modules/yardstick/config/ignite-failover-base-config.xml
@@ -0,0 +1,126 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    Ignite Spring configuration file to startup grid.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+    <bean id="base-ignite-failover.cfg" class="org.apache.ignite.configuration.IgniteConfiguration" abstract="true">
+        <property name="peerClassLoadingEnabled" value="false"/>
+
+        <property name="metricsLogFrequency" value="5000"/>
+
+        <property name="failureDetectionTimeout" value="2000"/>
+
+        <property name="cacheConfiguration">
+            <list>
+                <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="atomic">
+                    <property name="name" value="atomic-reties"/>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="atomic-offheap">
+                    <property name="name" value="atomic-offheap-reties"/>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="atomic">
+                    <property name="name" value="atomic-invoke-retry"/>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="atomic-offheap">
+                    <property name="name" value="atomic-offheap-invoke-retry"/>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="tx">
+                    <property name="name" value="tx-invoke-retry"/>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="tx-offheap">
+                    <property name="name" value="tx-offheap-invoke-retry"/>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="tx">
+                    <property name="name" value="tx-write-read"/>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="tx-offheap">
+                    <property name="name" value="tx-offheap-write-read"/>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="tx">
+                    <property name="name" value="tx-write-invoke"/>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="tx-offheap">
+                    <property name="name" value="tx-offheap-write-invoke"/>
+                </bean>
+            </list>
+        </property>
+
+        <property name="includeEventTypes">
+            <list/>
+        </property>
+
+        <property name="loadBalancingSpi">
+            <bean class="org.apache.ignite.spi.loadbalancing.roundrobin.RoundRobinLoadBalancingSpi">
+                <property name="perTask" value="false"/>
+            </bean>
+        </property>
+    </bean>
+
+    <bean id="atomic" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true">
+        <property name="cacheMode" value="PARTITIONED"/>
+
+        <property name="atomicityMode" value="ATOMIC"/>
+
+        <property name="swapEnabled" value="false"/>
+    </bean>
+
+    <bean id="atomic-offheap" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true">
+        <property name="name" value="atomic-offheap"/>
+
+        <property name="cacheMode" value="PARTITIONED"/>
+
+        <property name="atomicityMode" value="ATOMIC"/>
+
+        <property name="swapEnabled" value="false"/>
+
+        <property name="memoryMode" value="OFFHEAP_TIERED"/>
+    </bean>
+
+    <bean id="tx" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true">
+        <property name="cacheMode" value="PARTITIONED"/>
+
+        <property name="atomicityMode" value="TRANSACTIONAL"/>
+
+        <property name="swapEnabled" value="false"/>
+    </bean>
+
+    <bean id="tx-offheap" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true">
+        <property name="cacheMode" value="PARTITIONED"/>
+
+        <property name="atomicityMode" value="TRANSACTIONAL"/>
+
+        <property name="swapEnabled" value="false"/>
+
+        <property name="memoryMode" value="OFFHEAP_TIERED"/>
+    </bean>
+</beans>

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/config/ignite-failover-localhost-config.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-failover-localhost-config.xml b/modules/yardstick/config/ignite-failover-localhost-config.xml
new file mode 100644
index 0000000..a7c7ff8
--- /dev/null
+++ b/modules/yardstick/config/ignite-failover-localhost-config.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    Ignite Spring configuration file to startup grid.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+    <import resource="ignite-base-config.xml"/>
+    <import resource="ignite-failover-base-config.xml"/>
+
+    <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration" parent="base-ignite-failover.cfg">
+        <property name="localHost" value="127.0.0.1"/>
+
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+                        <property name="addresses">
+                            <list>
+                                <value>127.0.0.1:47500</value>
+                                <value>127.0.0.1:47501</value>
+                                <value>127.0.0.1:47502</value>
+                                <value>127.0.0.1:47503</value>
+                                <value>127.0.0.1:47504</value>
+                                <value>127.0.0.1:47505</value>
+                                <value>127.0.0.1:47506</value>
+                                <value>127.0.0.1:47507</value>
+                                <value>127.0.0.1:47508</value>
+                                <value>127.0.0.1:47509</value>
+                            </list>
+                        </property>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+    </bean>
+</beans>

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/pom.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index c87ecbc..5bb41b3 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -37,10 +37,9 @@
     <url>http://ignite.apache.org</url>
 
     <properties>
-        <yardstick.version>0.7.0</yardstick.version>
+        <yardstick.version>0.8.0</yardstick.version>
     </properties>
 
-
     <dependencies>
         <dependency>
             <groupId>org.apache.ignite</groupId>

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
index c9d4b85..fa93f00 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
@@ -131,4 +131,4 @@ public abstract class IgniteAbstractBenchmark extends BenchmarkDriverAdapter {
     protected int nextRandom(int min, int max) {
         return ThreadLocalRandom.current().nextInt(max - min) + min;
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
index 792d366..74b1da9 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
@@ -113,6 +113,26 @@ public class IgniteBenchmarkArguments {
     private String jdbcUrl;
 
     /** */
+    @Parameter(names = {"-rd", "--restartdelay"}, description = "Restart delay in seconds")
+    private int restartDelay = 20;
+
+    /** */
+    @Parameter(names = {"-rs", "--restartsleep"}, description = "Restart sleep in seconds")
+    private int restartSleep = 2;
+
+    /** */
+    @Parameter(names = {"-checkingPeriod", "--checkingPeriod"}, description = "Period to check cache consistency in seconds")
+    private int cacheConsistencyCheckingPeriod = 2 * 60;
+
+    /** */
+    @Parameter(names = {"-kc", "--keysCount"}, description = "Count of keys")
+    private int keysCnt = 5;
+
+    /** */
+    @Parameter(names = {"-cot", "--cacheOperationTimeout"}, description = "Max timeout for cache operations in seconds")
+    private int cacheOpTimeout = 30;
+
+    /** */
     @Parameter(names = {"-kpt", "--keysPerThread"}, description = "Use not intersecting keys in putAll benchmark")
     private boolean keysPerThread;
 
@@ -271,18 +291,54 @@ public class IgniteBenchmarkArguments {
     }
 
     /**
-     * @return Description.
+     * @return Delay in second which used in nodes restart algorithm.
      */
-    public String description() {
-        return "-nn=" + nodes + "-b=" + backups + "-sm=" + syncMode + "-cl=" + clientOnly + "-nc=" + nearCacheFlag +
-            (orderMode == null ? "" : "-wom=" + orderMode) + "-txc=" + txConcurrency;
+    public int restartDelay() {
+        return restartDelay;
+    }
+
+    /**
+     * @return Sleep in second which used in nodes restart algorithm.
+     */
+    public int restartSleep() {
+        return restartSleep;
+    }
+
+    /**
+     * @return Keys count.
+     */
+    public int keysCount() {
+        return keysCnt;
+    }
+
+    /**
+     * @return Period in seconds to check cache consistency.
+     */
+    public int cacheConsistencyCheckingPeriod() {
+        return cacheConsistencyCheckingPeriod;
+    }
+
+    /**
+     * @return Cache operation timeout in milliseconds.
+     */
+    public int cacheOperationTimeoutMillis() {
+        return cacheOpTimeout * 1000;
     }
 
     /**
      * @return {@code True} if use not intersecting keys in putAll benchmark.
      */
     public boolean keysPerThread() {
-       return keysPerThread;
+        return keysPerThread;
+    }
+
+    /**
+     * @return Description.
+     */
+    public String description() {
+        return "-nn=" + nodes + "-b=" + backups + "-sm=" + syncMode + "-cl=" + clientOnly + "-nc=" + nearCacheFlag +
+            (orderMode == null ? "" : "-wom=" + orderMode) + "-txc=" + txConcurrency + "-rd=" + restartDelay +
+            "-rs=" + restartSleep;
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
index 22a9eac..3efa4a5 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
@@ -27,9 +27,9 @@ import org.yardstickframework.BenchmarkUtils;
 /**
  * Abstract class for Ignite benchmarks which use cache.
  */
-public abstract class IgniteCacheAbstractBenchmark extends IgniteAbstractBenchmark {
+public abstract class IgniteCacheAbstractBenchmark<K, V> extends IgniteAbstractBenchmark {
     /** Cache. */
-    protected IgniteCache<Integer, Object> cache;
+    protected IgniteCache<K, V> cache;
 
     /** */
     private ThreadLocal<ThreadRange> threadRange = new ThreadLocal<>();
@@ -77,7 +77,7 @@ public abstract class IgniteCacheAbstractBenchmark extends IgniteAbstractBenchma
      *
      * @return IgniteCache Cache to use.
      */
-    protected abstract IgniteCache<Integer, Object> cache();
+    protected abstract IgniteCache<K, V> cache();
 
     /**
      *
@@ -109,4 +109,4 @@ public abstract class IgniteCacheAbstractBenchmark extends IgniteAbstractBenchma
             return rnd.nextInt(min, max);
         }
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
index e023c1d..8a86e2f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
@@ -23,7 +23,7 @@ import org.apache.ignite.IgniteCache;
 /**
  * Ignite benchmark that performs get operations.
  */
-public class IgniteGetBenchmark extends IgniteCacheAbstractBenchmark {
+public class IgniteGetBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** {@inheritDoc} */
     @Override public boolean test(Map<Object, Object> ctx) throws Exception {
         int key = nextRandom(args.range());
@@ -37,4 +37,4 @@ public class IgniteGetBenchmark extends IgniteCacheAbstractBenchmark {
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("atomic");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteJdbcSqlQueryBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteJdbcSqlQueryBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteJdbcSqlQueryBenchmark.java
index 63c274f..bbdd760 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteJdbcSqlQueryBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteJdbcSqlQueryBenchmark.java
@@ -37,7 +37,7 @@ import static org.yardstickframework.BenchmarkUtils.println;
 /**
  * Ignite benchmark that performs query operations.
  */
-public class IgniteJdbcSqlQueryBenchmark extends IgniteCacheAbstractBenchmark {
+public class IgniteJdbcSqlQueryBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** Statements for closing. */
     Set<PreparedStatement> stms = Collections.synchronizedSet(new HashSet<PreparedStatement>());
 
@@ -136,4 +136,4 @@ public class IgniteJdbcSqlQueryBenchmark extends IgniteCacheAbstractBenchmark {
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("query");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllBenchmark.java
index 2ce707c..8cd2347 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllBenchmark.java
@@ -28,7 +28,7 @@ import org.yardstickframework.BenchmarkConfiguration;
 /**
  * Ignite benchmark that performs putAll operations.
  */
-public class IgnitePutAllBenchmark extends IgniteCacheAbstractBenchmark {
+public class IgnitePutAllBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** Affinity mapper. */
     private Affinity<Integer> aff;
 
@@ -65,4 +65,4 @@ public class IgnitePutAllBenchmark extends IgniteCacheAbstractBenchmark {
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("atomic");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllTxBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllTxBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllTxBenchmark.java
index 32d5b02..63faa2f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllTxBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutAllTxBenchmark.java
@@ -28,7 +28,7 @@ import org.yardstickframework.BenchmarkConfiguration;
 /**
  * Ignite benchmark that performs transactional putAll operations.
  */
-public class IgnitePutAllTxBenchmark extends IgniteCacheAbstractBenchmark {
+public class IgnitePutAllTxBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** Affinity mapper. */
     private Affinity<Integer> aff;
 
@@ -68,4 +68,4 @@ public class IgnitePutAllTxBenchmark extends IgniteCacheAbstractBenchmark {
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("tx");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutBenchmark.java
index ba2d959..69db87f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutBenchmark.java
@@ -24,7 +24,7 @@ import org.apache.ignite.yardstick.cache.model.SampleValue;
 /**
  * Ignite benchmark that performs put operations.
  */
-public class IgnitePutBenchmark extends IgniteCacheAbstractBenchmark {
+public class IgnitePutBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** {@inheritDoc} */
     @Override public boolean test(Map<Object, Object> ctx) throws Exception {
         int key = nextRandom(args.range());
@@ -38,4 +38,4 @@ public class IgnitePutBenchmark extends IgniteCacheAbstractBenchmark {
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("atomic");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutGetBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutGetBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutGetBenchmark.java
index a92a35e..42f308c 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutGetBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutGetBenchmark.java
@@ -24,7 +24,7 @@ import org.apache.ignite.yardstick.cache.model.SampleValue;
 /**
  * Ignite benchmark that performs put and get operations.
  */
-public class IgnitePutGetBenchmark extends IgniteCacheAbstractBenchmark {
+public class IgnitePutGetBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** {@inheritDoc} */
     @Override public boolean test(Map<Object, Object> ctx) throws Exception {
         int key = nextRandom(args.range());
@@ -43,4 +43,4 @@ public class IgnitePutGetBenchmark extends IgniteCacheAbstractBenchmark {
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("atomic");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutGetTxBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutGetTxBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutGetTxBenchmark.java
index 07b80cc..5afe0b2 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutGetTxBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutGetTxBenchmark.java
@@ -25,7 +25,7 @@ import org.apache.ignite.yardstick.cache.model.SampleValue;
 /**
  * Ignite benchmark that performs transactional put and get operations.
  */
-public class IgnitePutGetTxBenchmark extends IgniteCacheAbstractBenchmark {
+public class IgnitePutGetTxBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** {@inheritDoc} */
     @Override public boolean test(Map<Object, Object> ctx) throws Exception {
         int key = nextRandom(0, args.range() / 2);
@@ -48,4 +48,4 @@ public class IgnitePutGetTxBenchmark extends IgniteCacheAbstractBenchmark {
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("tx");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue1Benchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue1Benchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue1Benchmark.java
index 47b6922..6f06015 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue1Benchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue1Benchmark.java
@@ -24,7 +24,7 @@ import org.apache.ignite.yardstick.cache.model.Person1;
 /**
  * Ignite benchmark that performs put operations for entity with indexed fields.
  */
-public class IgnitePutIndexedValue1Benchmark extends IgniteCacheAbstractBenchmark {
+public class IgnitePutIndexedValue1Benchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** {@inheritDoc} */
     @Override public boolean test(Map<Object, Object> ctx) throws Exception {
         int key = nextRandom(args.range());
@@ -38,4 +38,4 @@ public class IgnitePutIndexedValue1Benchmark extends IgniteCacheAbstractBenchmar
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("atomic-index");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue2Benchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue2Benchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue2Benchmark.java
index 69d603c..0112163 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue2Benchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue2Benchmark.java
@@ -24,7 +24,7 @@ import org.apache.ignite.yardstick.cache.model.Person2;
 /**
  * Ignite benchmark that performs put operations for entity with indexed fields.
  */
-public class IgnitePutIndexedValue2Benchmark extends IgniteCacheAbstractBenchmark {
+public class IgnitePutIndexedValue2Benchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** {@inheritDoc} */
     @Override public boolean test(Map<Object, Object> ctx) throws Exception {
         int key = nextRandom(args.range());
@@ -38,4 +38,4 @@ public class IgnitePutIndexedValue2Benchmark extends IgniteCacheAbstractBenchmar
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("atomic-index");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue8Benchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue8Benchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue8Benchmark.java
index fb5dd68..dae32b4 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue8Benchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutIndexedValue8Benchmark.java
@@ -24,7 +24,7 @@ import org.apache.ignite.yardstick.cache.model.Person8;
 /**
  * Ignite benchmark that performs put operations for entity with indexed fields.
  */
-public class IgnitePutIndexedValue8Benchmark extends IgniteCacheAbstractBenchmark {
+public class IgnitePutIndexedValue8Benchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** {@inheritDoc} */
     @Override public boolean test(Map<Object, Object> ctx) throws Exception {
         int key = nextRandom(args.range());
@@ -38,4 +38,4 @@ public class IgnitePutIndexedValue8Benchmark extends IgniteCacheAbstractBenchmar
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("atomic-index");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
index 53bd60c..ead3a63 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
@@ -24,7 +24,7 @@ import org.apache.ignite.yardstick.cache.model.SampleValue;
 /**
  * Ignite benchmark that performs transactional put operations.
  */
-public class IgnitePutTxBenchmark extends IgniteCacheAbstractBenchmark {
+public class IgnitePutTxBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** {@inheritDoc} */
     @Override public boolean test(Map<Object, Object> ctx) throws Exception {
         int key = nextRandom(args.range());
@@ -39,4 +39,4 @@ public class IgnitePutTxBenchmark extends IgniteCacheAbstractBenchmark {
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("tx");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryBenchmark.java
index fa16cf2..8e31455 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryBenchmark.java
@@ -32,7 +32,7 @@ import static org.yardstickframework.BenchmarkUtils.println;
 /**
  * Ignite benchmark that performs query operations.
  */
-public class IgniteSqlQueryBenchmark extends IgniteCacheAbstractBenchmark {
+public class IgniteSqlQueryBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** {@inheritDoc} */
     @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
         super.setUp(cfg);
@@ -90,4 +90,4 @@ public class IgniteSqlQueryBenchmark extends IgniteCacheAbstractBenchmark {
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("query");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryJoinBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryJoinBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryJoinBenchmark.java
index ad863b8..1f8006d 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryJoinBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryJoinBenchmark.java
@@ -33,7 +33,7 @@ import static org.yardstickframework.BenchmarkUtils.println;
 /**
  * Ignite benchmark that performs query operations with joins.
  */
-public class IgniteSqlQueryJoinBenchmark extends IgniteCacheAbstractBenchmark {
+public class IgniteSqlQueryJoinBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** {@inheritDoc} */
     @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
         super.setUp(cfg);
@@ -116,4 +116,4 @@ public class IgniteSqlQueryJoinBenchmark extends IgniteCacheAbstractBenchmark {
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("query");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutBenchmark.java
index ea9531a..1c258a4 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutBenchmark.java
@@ -29,7 +29,7 @@ import org.yardstickframework.BenchmarkConfiguration;
 /**
  * Ignite benchmark that performs put and query operations.
  */
-public class IgniteSqlQueryPutBenchmark extends IgniteCacheAbstractBenchmark {
+public class IgniteSqlQueryPutBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
     /** {@inheritDoc} */
     @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
         super.setUp(cfg);
@@ -81,4 +81,4 @@ public class IgniteSqlQueryPutBenchmark extends IgniteCacheAbstractBenchmark {
     @Override protected IgniteCache<Integer, Object> cache() {
         return ignite().cache("query");
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicInvokeRetryBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicInvokeRetryBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicInvokeRetryBenchmark.java
new file mode 100644
index 0000000..c0567ef
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicInvokeRetryBenchmark.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.failover;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.cache.CacheEntryProcessor;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.yardstickframework.BenchmarkConfiguration;
+
+import static org.yardstickframework.BenchmarkUtils.println;
+
+/**
+ * Invoke retry failover benchmark. <p> Each client maintains a local map that it updates together with cache. Client
+ * invokes an increment closure for all generated keys and atomically increments value for corresponding keys in the
+ * local map. To validate cache contents, all writes from the client are stopped, values in the local map are compared
+ * to the values in the cache.
+ */
+public class IgniteAtomicInvokeRetryBenchmark extends IgniteFailoverAbstractBenchmark<String, Set> {
+    /** */
+    private final ConcurrentMap<String, AtomicLong> nextValMap = new ConcurrentHashMap<>();
+
+    /** */
+    private final ReadWriteLock rwl = new ReentrantReadWriteLock(true);
+
+    /** */
+    private volatile Exception ex;
+
+    /** {@inheritDoc} */
+    @Override public void setUp(final BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        Thread thread = new Thread(new Runnable() {
+            @Override public void run() {
+                try {
+                    final int timeout = args.cacheOperationTimeoutMillis();
+                    final int range = args.range();
+
+                    while (!Thread.currentThread().isInterrupted()) {
+                        Thread.sleep(args.cacheConsistencyCheckingPeriod() * 1000);
+
+                        rwl.writeLock().lock();
+
+                        try {
+                            println("Start cache validation.");
+
+                            long startTime = U.currentTimeMillis();
+
+                            Map<String, Set> badCacheEntries = new HashMap<>();
+
+                            for (Map.Entry<String, AtomicLong> e : nextValMap.entrySet()) {
+                                String key = e.getKey();
+
+                                asyncCache.get(key);
+                                Set set = asyncCache.<Set>future().get(timeout);
+
+                                if (set == null || e.getValue() == null || !Objects.equals(e.getValue().get(), (long)set.size()))
+                                    badCacheEntries.put(key, set);
+                            }
+
+                            if (!badCacheEntries.isEmpty()) {
+                                // Print all usefull information and finish.
+                                for (Map.Entry<String, Set> e : badCacheEntries.entrySet()) {
+                                    String key = e.getKey();
+
+                                    println("Got unexpected set size [key='" + key + "', expSize=" + nextValMap.get(key)
+                                        + ", cacheVal=" + e.getValue() + "]");
+                                }
+
+                                println("Next values map contant:");
+                                for (Map.Entry<String, AtomicLong> e : nextValMap.entrySet())
+                                    println("Map Entry [key=" + e.getKey() + ", val=" + e.getValue() + "]");
+
+                                println("Cache content:");
+
+                                for (int k2 = 0; k2 < range; k2++) {
+                                    String key2 = "key-" + k2;
+
+                                    asyncCache.get(key2);
+                                    Object val = asyncCache.future().get(timeout);
+
+                                    if (val != null)
+                                        println("Cache Entry [key=" + key2 + ", val=" + val + "]");
+
+                                }
+
+                                throw new IllegalStateException("Cache and local map are in inconsistent state " +
+                                    "[badKeys=" + badCacheEntries.keySet() + ']');
+                            }
+
+                            println("Clearing all data.");
+
+                            asyncCache.removeAll();
+                            asyncCache.future().get(timeout);
+
+                            nextValMap.clear();
+
+                            println("Cache validation successfully finished in "
+                                + (U.currentTimeMillis() - startTime) / 1000 + " sec.");
+                        }
+                        finally {
+                            rwl.writeLock().unlock();
+                        }
+                    }
+                }
+                catch (Throwable e) {
+                    ex = new Exception(e);
+
+                    println("Got exception: " + e);
+
+                    e.printStackTrace();
+
+                    if (e instanceof Error)
+                        throw (Error)e;
+                }
+            }
+        }, "cache-" + cacheName() + "-validator");
+
+        thread.setDaemon(true);
+
+        thread.start();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        final int k = nextRandom(args.range());
+
+        String key = "key-" + k;
+
+        rwl.readLock().lock();
+
+        try {
+            if (ex != null)
+                throw ex;
+
+            AtomicLong nextAtomicVal = nextValMap.putIfAbsent(key, new AtomicLong(1));
+
+            Long nextVal = 1L;
+
+            if (nextAtomicVal != null)
+                nextVal = nextAtomicVal.incrementAndGet();
+
+            asyncCache.invoke(key, new AddInSetEntryProcessor(), nextVal);
+            asyncCache.future().get(args.cacheOperationTimeoutMillis());
+        }
+        finally {
+            rwl.readLock().unlock();
+        }
+
+        if (ex != null)
+            throw ex;
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected String cacheName() {
+        return "atomic-invoke-retry";
+    }
+
+    /**
+     */
+    private static class AddInSetEntryProcessor implements CacheEntryProcessor<String, Set, Object> {
+        /** */
+        private static final long serialVersionUID = 0;
+
+        /** {@inheritDoc} */
+        @Override public Object process(MutableEntry<String, Set> entry,
+            Object... arguments) throws EntryProcessorException {
+            assert !F.isEmpty(arguments);
+
+            Object val = arguments[0];
+
+            Set set;
+
+            if (!entry.exists())
+                set = new HashSet<>();
+            else
+                set = entry.getValue();
+
+            set.add(val);
+
+            entry.setValue(set);
+
+            return null;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicOffHeapInvokeRetryBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicOffHeapInvokeRetryBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicOffHeapInvokeRetryBenchmark.java
new file mode 100644
index 0000000..c8b0b1d
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicOffHeapInvokeRetryBenchmark.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.failover;
+
+/**
+ * Invoke retry failover benchmark. <p> Each client maintains a local map that it updates together with cache. Client
+ * invokes an increment closure for all generated keys and atomically increments value for corresponding keys in the
+ * local map. To validate cache contents, all writes from the client are stopped, values in the local map are compared
+ * to the values in the cache.
+ */
+public class IgniteAtomicOffHeapInvokeRetryBenchmark extends IgniteAtomicInvokeRetryBenchmark {
+    /** {@inheritDoc} */
+    @Override protected String cacheName() {
+        return "atomic-offheap-invoke-retry";
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicOffHeapRetriesBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicOffHeapRetriesBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicOffHeapRetriesBenchmark.java
new file mode 100644
index 0000000..ebb9eac
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicOffHeapRetriesBenchmark.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.failover;
+
+/**
+ * Atomic retries failover benchmark.
+ * <p>
+ * Client generates continuous load to the cluster (random get, put, invoke, remove
+ * operations).
+ */
+public class IgniteAtomicOffHeapRetriesBenchmark extends IgniteAtomicRetriesBenchmark {
+    /** {@inheritDoc} */
+    @Override protected String cacheName() {
+        return "atomic-offheap-reties";
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicRetriesBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicRetriesBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicRetriesBenchmark.java
new file mode 100644
index 0000000..4e60698
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteAtomicRetriesBenchmark.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.failover;
+
+import java.util.Map;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.cache.CacheEntryProcessor;
+
+/**
+ * Atomic retries failover benchmark.
+ * <p>
+ * Client generates continuous load to the cluster (random get, put, invoke, remove
+ * operations).
+ */
+public class IgniteAtomicRetriesBenchmark extends IgniteFailoverAbstractBenchmark<Integer, String> {
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        final int key = nextRandom(args.range());
+
+        int opNum = nextRandom(4);
+
+        final int timeout = args.cacheOperationTimeoutMillis();
+
+        switch (opNum) {
+            case 0:
+                asyncCache.get(key);
+                asyncCache.future().get(timeout);
+
+                break;
+
+            case 1:
+                asyncCache.put(key, String.valueOf(key));
+                asyncCache.future().get(timeout);
+
+                break;
+
+            case 2:
+                asyncCache.invoke(key, new TestCacheEntryProcessor());
+                asyncCache.future().get(timeout);
+
+                break;
+
+            case 3:
+                asyncCache.remove(key);
+                asyncCache.future().get(timeout);
+
+                break;
+
+            default:
+                throw new IllegalStateException("Got invalid operation number: " + opNum);
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected String cacheName() {
+        return "atomic-reties";
+    }
+
+    /**
+     */
+    private static class TestCacheEntryProcessor implements CacheEntryProcessor<Integer, String, String> {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0;
+
+        /** {@inheritDoc} */
+        @Override public String process(MutableEntry<Integer, String> entry,
+            Object... arguments) throws EntryProcessorException {
+            return "key";
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteFailoverAbstractBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteFailoverAbstractBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteFailoverAbstractBenchmark.java
new file mode 100644
index 0000000..83fc58f
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteFailoverAbstractBenchmark.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.failover;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.mxbean.IgniteMXBean;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.yardstick.cache.IgniteCacheAbstractBenchmark;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+import org.yardstickframework.BenchmarkUtils.ProcessExecutionResult;
+
+import static org.yardstickframework.BenchmarkUtils.println;
+
+/**
+ * Ignite benchmark that performs long running failover tasks.
+ */
+public abstract class IgniteFailoverAbstractBenchmark<K, V> extends IgniteCacheAbstractBenchmark<K, V> {
+    /** */
+    private static final AtomicBoolean restarterStarted = new AtomicBoolean();
+
+    /** Async Cache. */
+    protected IgniteCache<K, V> asyncCache;
+
+    /** */
+    private final AtomicBoolean firtsExProcessed = new AtomicBoolean();
+
+    /** {@inheritDoc} */
+    @Override public void setUp(final BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        asyncCache = cache.withAsync();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onWarmupFinished() {
+        if (cfg.memberId() == 0 && restarterStarted.compareAndSet(false, true)) {
+            Thread restarterThread = new Thread(new Runnable() {
+                @Override public void run() {
+                    try {
+                        println("Servers restarter started on driver: "
+                            + IgniteFailoverAbstractBenchmark.this.getClass().getSimpleName());
+
+                        Ignite ignite = ignite();
+
+                        // Read servers configs from cache to local map.
+                        IgniteCache<Integer, BenchmarkConfiguration> srvsCfgsCache = ignite.
+                            getOrCreateCache(new CacheConfiguration<Integer, BenchmarkConfiguration>().
+                                setName("serversConfigs"));
+
+                        final Map<Integer, BenchmarkConfiguration> srvsCfgs = new HashMap<>();
+
+                        for (Cache.Entry<Integer, BenchmarkConfiguration> e : srvsCfgsCache) {
+                            println("Read entry from 'serversConfigs' cache : " + e);
+
+                            srvsCfgs.put(e.getKey(), e.getValue());
+                        }
+
+                        assert ignite.cluster().forServers().nodes().size() == srvsCfgs.size();
+
+                        final int backupsCnt = args.backups();
+
+                        assert backupsCnt >= 1 : "Backups: " + backupsCnt;
+
+                        final boolean isDebug = ignite.log().isDebugEnabled();
+
+                        // Main logic.
+                        while (!Thread.currentThread().isInterrupted()) {
+                            Thread.sleep(args.restartDelay() * 1000);
+
+                            int numNodesToRestart = nextRandom(1, backupsCnt + 1);
+
+                            List<Integer> ids = new ArrayList<>();
+
+                            ids.addAll(srvsCfgs.keySet());
+
+                            Collections.shuffle(ids);
+
+                            println("Waiting for partitioned map exchage of all nodes");
+
+                            IgniteCompute asyncCompute = ignite.compute().withAsync();
+
+                            asyncCompute.broadcast(new AwaitPartitionMapExchangeTask());
+
+                            asyncCompute.future().get(args.cacheOperationTimeoutMillis());
+
+                            println("Start servers restarting [numNodesToRestart=" + numNodesToRestart
+                                + ", shuffledIds=" + ids + "]");
+
+                            for (int i = 0; i < numNodesToRestart; i++) {
+                                Integer id = ids.get(i);
+
+                                BenchmarkConfiguration bc = srvsCfgs.get(id);
+
+                                ProcessExecutionResult res = BenchmarkUtils.kill9Server(bc, isDebug);
+
+                                println("Server with id " + id + " has been killed."
+                                    + (isDebug ? " Process execution result:\n" + res : ""));
+                            }
+
+                            Thread.sleep(args.restartSleep() * 1000);
+
+                            for (int i = 0; i < numNodesToRestart; i++) {
+                                Integer id = ids.get(i);
+
+                                BenchmarkConfiguration bc = srvsCfgs.get(id);
+
+                                ProcessExecutionResult res = BenchmarkUtils.startServer(bc, isDebug);
+
+                                println("Server with id " + id + " has been started."
+                                    + (isDebug ? " Process execution result:\n" + res : ""));
+                            }
+                        }
+                    }
+                    catch (Throwable e) {
+                        println("Got exception: " + e);
+                        e.printStackTrace();
+
+                        U.dumpThreads(null);
+
+                        if (e instanceof Error)
+                            throw (Error)e;
+                    }
+                }
+            }, "servers-restarter");
+
+            restarterThread.setDaemon(true);
+            restarterThread.start();
+        }
+    }
+
+    /**
+     * Awaits for partitiona map exchage.
+     *
+     * @param ignite Ignite.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("BusyWait")
+    protected static void awaitPartitionMapExchange(Ignite ignite) throws Exception {
+        IgniteLogger log = ignite.log();
+
+        log.info("Waiting for finishing of a partition exchange on node: " + ignite);
+
+        IgniteKernal kernal = (IgniteKernal)ignite;
+
+        while (true) {
+            boolean partitionsExchangeFinished = true;
+
+            for (IgniteInternalCache<?, ?> cache : kernal.cachesx(null)) {
+                log.info("Checking cache: " + cache.name());
+
+                GridCacheAdapter<?, ?> c = kernal.internalCache(cache.name());
+
+                if (!(c instanceof GridDhtCacheAdapter))
+                    break;
+
+                GridDhtCacheAdapter<?, ?> dht = (GridDhtCacheAdapter<?, ?>)c;
+
+                GridDhtPartitionFullMap partMap = dht.topology().partitionMap(true);
+
+                for (Map.Entry<UUID, GridDhtPartitionMap> e : partMap.entrySet()) {
+                    log.info("Checking node: " + e.getKey());
+
+                    for (Map.Entry<Integer, GridDhtPartitionState> e1 : e.getValue().entrySet()) {
+                        if (e1.getValue() != GridDhtPartitionState.OWNING) {
+                            log.info("Undesired state [id=" + e1.getKey() + ", state=" + e1.getValue() + ']');
+
+                            partitionsExchangeFinished = false;
+
+                            break;
+                        }
+                    }
+
+                    if (!partitionsExchangeFinished)
+                        break;
+                }
+
+                if (!partitionsExchangeFinished)
+                    break;
+            }
+
+            if (partitionsExchangeFinished)
+                return;
+
+            Thread.sleep(100);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onException(Throwable e) {
+        // Proceess only the first exception to prevent a multiple printing of a full thread dump.
+        if (firtsExProcessed.compareAndSet(false, true)) {
+            // Debug info on current client.
+            println("Full thread dump of the current node below.");
+
+            U.dumpThreads(null);
+
+            println("");
+
+            ((IgniteMXBean)ignite()).dumpDebugInfo();
+
+            // Debug info on servers.
+            Ignite ignite = ignite();
+
+            ClusterGroup srvs = ignite.cluster().forServers();
+
+            IgniteCompute asyncCompute = ignite.compute(srvs).withAsync();
+
+            asyncCompute.broadcast(new ThreadDumpPrinterTask(ignite.cluster().localNode().id(), e));
+            asyncCompute.future().get(10_000);
+        }
+    }
+
+    /**
+     * @return Cache name.
+     */
+    protected abstract String cacheName();
+
+    /** {@inheritDoc} */
+    @Override protected IgniteCache<K, V> cache() {
+        return ignite().cache(cacheName());
+    }
+
+    /**
+     */
+    private static class ThreadDumpPrinterTask implements IgniteRunnable {
+        /** */
+        private static final long serialVersionUID = 0;
+
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** */
+        private final UUID id;
+
+        /** */
+        private final Throwable e;
+
+        /**
+         * @param id Benchmark node id.
+         * @param e Exception.
+         */
+        ThreadDumpPrinterTask(UUID id, Throwable e) {
+            this.id = id;
+            this.e = e;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            println("Driver finished with exception [driverNodeId=" + id + ", e=" + e + "]");
+            println("Full thread dump of the current server node below.");
+
+            U.dumpThreads(null);
+
+            println("");
+
+            ((IgniteMXBean)ignite).dumpDebugInfo();
+        }
+    }
+
+    /**
+     */
+    private static class AwaitPartitionMapExchangeTask implements IgniteRunnable {
+        /** */
+        private static final long serialVersionUID = 0;
+
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            try {
+                awaitPartitionMapExchange(ignite);
+            }
+            catch (Exception e) {
+                throw new IgniteException(e);
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/5b0a18dd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteFailoverNode.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteFailoverNode.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteFailoverNode.java
new file mode 100644
index 0000000..29405de
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteFailoverNode.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache.failover;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.util.List;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.yardstick.IgniteNode;
+import org.yardstickframework.BenchmarkConfiguration;
+
+import static org.yardstickframework.BenchmarkUtils.println;
+
+/**
+ * Ignite failover node.
+ */
+public class IgniteFailoverNode extends IgniteNode {
+    /** {@inheritDoc} */
+    @Override public void start(BenchmarkConfiguration cfg) throws Exception {
+        super.start(cfg);
+
+        // Put server configuration at special cache.
+        RuntimeMXBean mxBean = ManagementFactory.getRuntimeMXBean();
+
+        List<String> jvmOpts = mxBean.getInputArguments();
+
+        StringBuilder jvmOptsStr = new StringBuilder();
+
+        for (String opt : jvmOpts)
+            jvmOptsStr.append(opt).append(' ');
+
+        cfg.customProperties().put("JVM_OPTS", jvmOptsStr.toString());
+        cfg.customProperties().put("PROPS_ENV", System.getenv("PROPS_ENV"));
+        cfg.customProperties().put("CLASSPATH", mxBean.getClassPath());
+        cfg.customProperties().put("JAVA", System.getenv("JAVA"));
+
+        IgniteCache<Integer, BenchmarkConfiguration> srvsCfgsCache = ignite().
+            getOrCreateCache(new CacheConfiguration<Integer, BenchmarkConfiguration>().setName("serversConfigs"));
+
+        srvsCfgsCache.put(cfg.memberId(), cfg);
+
+        println("Put at cache [" + cfg.memberId() + "=" + cfg + "]");
+    }
+}


Mime
View raw message