Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 61DF72007D1 for ; Thu, 12 May 2016 08:42:03 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 60D59160A10; Thu, 12 May 2016 06:42:03 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D4451160A26 for ; Thu, 12 May 2016 08:42:00 +0200 (CEST) Received: (qmail 76114 invoked by uid 500); 12 May 2016 06:42:00 -0000 Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.apache.org Delivered-To: mailing list commits@ignite.apache.org Received: (qmail 74581 invoked by uid 99); 12 May 2016 06:41:56 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 May 2016 06:41:56 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7378EE032D; Thu, 12 May 2016 06:41:56 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: akuznetsov@apache.org To: commits@ignite.apache.org Date: Thu, 12 May 2016 06:42:30 -0000 Message-Id: <1160776e74cf46ee91b741e9e45aa326@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [36/60] [abbrv] ignite git commit: IGNITE-3051 Yardstick load tests: continious query archived-at: Thu, 12 May 2016 06:42:03 -0000 IGNITE-3051 Yardstick load tests: continious query IGNITE-3050 Yardstick load tests: add a filter option for allowed operations pIGNITE-3049 Yardstick sql load tests should read sql quiers from an external file Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/339df384 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/339df384 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/339df384 Branch: refs/heads/ignite-2832 Commit: 339df384d3fffa724d8c18dd952a9e2a3a8f01c1 Parents: 4b11219 Author: Vladislav Pyatkov Authored: Fri May 6 15:52:01 2016 +0300 Committer: Denis Magda Committed: Fri May 6 15:52:01 2016 +0300 ---------------------------------------------------------------------- .../config/benchmark-cache-load-win.properties | 4 +- .../config/benchmark-cache-load.properties | 2 +- .../config/ignite-base-load-config.xml | 63 ++++---- .../config/ignite-cache-load-config.xml | 1 + modules/yardstick/config/queries.txt | 2 + .../yardstick/IgniteBenchmarkArguments.java | 26 ++++ .../IgniteCacheRandomOperationBenchmark.java | 148 ++++++++++++++++++- .../yardstick/cache/load/model/ModelUtil.java | 4 +- 8 files changed, 207 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/339df384/modules/yardstick/config/benchmark-cache-load-win.properties ---------------------------------------------------------------------- diff --git a/modules/yardstick/config/benchmark-cache-load-win.properties b/modules/yardstick/config/benchmark-cache-load-win.properties index bfcde77..7469c01 100644 --- a/modules/yardstick/config/benchmark-cache-load-win.properties +++ b/modules/yardstick/config/benchmark-cache-load-win.properties @@ -53,4 +53,6 @@ set DRIVER_HOSTS=localhost :: Run configuration which contains all benchmarks. :: Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute). set CONFIGS=^ --cfg %SCRIPT_DIR%\..\config\ignite-cache-load-config.xml -r 10000 -pa 500 -bs 50 -b 1 -w 20 -d 30 -t 64 -sm PRIMARY_SYNC -dn IgniteCacheRandomOperationBenchmark -sn IgniteNode -ds atomic-get-1-backup -nn 1 +-cfg %SCRIPT_DIR%\..\config\ignite-cache-load-config.xml -r 10000 -pa 500 -bs 50 -b 1 -w 20 -d 30 -t 64 -sm PRIMARY_SYNC -dn IgniteCacheRandomOperationBenchmark -sn IgniteNode -ds cache-random-benchmark-1-backup -nn 1^ + -qf config\queries.txt --allow-operation PUT --allow-operation PUT_ALL --allow-operation GET --allow-operation GET_ALL --allow-operation INVOKE --allow-operation INVOKE_ALL --allow-operation REMOVE^ + --allow-operation REMOVE_ALL --allow-operation PUT_IF_ABSENT --allow-operation REPLACE --allow-operation SCAN_QUERY --allow-operation SQL_QUERY --allow-operation CONTINUOUS_QUERIE http://git-wip-us.apache.org/repos/asf/ignite/blob/339df384/modules/yardstick/config/benchmark-cache-load.properties ---------------------------------------------------------------------- diff --git a/modules/yardstick/config/benchmark-cache-load.properties b/modules/yardstick/config/benchmark-cache-load.properties index 3604f25..3d9d4a6 100644 --- a/modules/yardstick/config/benchmark-cache-load.properties +++ b/modules/yardstick/config/benchmark-cache-load.properties @@ -83,4 +83,4 @@ r=1000000 # Run configuration. # Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute). CONFIGS="\ --cfg ${SCRIPT_DIR}/../config/ignite-cache-load-config.xml -nn ${nodesNum} -b ${b} -r ${r} -w ${w} -d ${d} -pa ${pa} -t ${t} -sm ${sm} -dn IgniteCacheRandomOperationBenchmark -sn IgniteNode -ds cache-random-benchmark-1-backup" \ No newline at end of file +-cfg ${SCRIPT_DIR}/../config/ignite-cache-load-config.xml -nn ${nodesNum} -b ${b} -r ${r} -w ${w} -d ${d} -pa ${pa} -t ${t} -sm ${sm} -dn IgniteCacheRandomOperationBenchmark -sn IgniteNode -ds cache-random-benchmark-1-backup -qf config\queries.txt" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/339df384/modules/yardstick/config/ignite-base-load-config.xml ---------------------------------------------------------------------- diff --git a/modules/yardstick/config/ignite-base-load-config.xml b/modules/yardstick/config/ignite-base-load-config.xml index 1ec1e23..3c13180 100644 --- a/modules/yardstick/config/ignite-base-load-config.xml +++ b/modules/yardstick/config/ignite-base-load-config.xml @@ -1,23 +1,6 @@ - - - + + + id + salary + + + + + + + + + + + + + + + + + + - + - - - + @@ -205,15 +211,8 @@ - - - - id - salary - - - + http://git-wip-us.apache.org/repos/asf/ignite/blob/339df384/modules/yardstick/config/ignite-cache-load-config.xml ---------------------------------------------------------------------- diff --git a/modules/yardstick/config/ignite-cache-load-config.xml b/modules/yardstick/config/ignite-cache-load-config.xml index e14247a..6c6ac89 100644 --- a/modules/yardstick/config/ignite-cache-load-config.xml +++ b/modules/yardstick/config/ignite-cache-load-config.xml @@ -39,6 +39,7 @@ + http://git-wip-us.apache.org/repos/asf/ignite/blob/339df384/modules/yardstick/config/queries.txt ---------------------------------------------------------------------- diff --git a/modules/yardstick/config/queries.txt b/modules/yardstick/config/queries.txt new file mode 100644 index 0000000..8271100 --- /dev/null +++ b/modules/yardstick/config/queries.txt @@ -0,0 +1,2 @@ +SELECT Person.firstName FROM "query".Person, "orgCache".Organization WHERE Person.orgId = Organization.id AND lower(Organization.name) = lower('Organization 55') +SELECT Organization.name FROM "orgCache".Organization WHERE lower(Organization.name) LIKE lower('%55%') http://git-wip-us.apache.org/repos/asf/ignite/blob/339df384/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java ---------------------------------------------------------------------- diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java index 4531ed3..d210644 100644 --- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java @@ -24,6 +24,9 @@ import org.apache.ignite.internal.util.tostring.GridToStringBuilder; import org.apache.ignite.transactions.TransactionConcurrency; import org.apache.ignite.transactions.TransactionIsolation; +import java.util.ArrayList; +import java.util.List; + /** * Input arguments for Ignite benchmarks. */ @@ -42,6 +45,11 @@ public class IgniteBenchmarkArguments { private String cfg = "config/ignite-localhost-config.xml"; /** */ + @Parameter(names = {"-qf", "--queriesFile"}, description = "File with the list of SQL queries " + + "predefined for the load test") + private String qFile = null; + + /** */ @Parameter(names = {"-sm", "--syncMode"}, description = "Synchronization mode") private CacheWriteSynchronizationMode syncMode = CacheWriteSynchronizationMode.PRIMARY_SYNC; @@ -145,6 +153,17 @@ public class IgniteBenchmarkArguments { @Parameter(names = {"-pp", "--printPartitionStats"}, description = "Print partition statistics") private boolean printPartStats; + /** */ + @Parameter(names = "--allow-operation", description = "List of allowed load test cache operations") + private List allowedOperations = new ArrayList<>(); + + /** + * @return List of cache operations. + */ + public List allowOperations() { + return allowedOperations; + } + /** * @return If {@code true} when need to print partition statistics. */ @@ -279,6 +298,13 @@ public class IgniteBenchmarkArguments { } /** + * @return File contains SQL queries. + */ + public String queriesFile() { + return qFile; + } + + /** * @return Number of jobs */ public int jobs() { http://git-wip-us.apache.org/repos/asf/ignite/blob/339df384/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java ---------------------------------------------------------------------- diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java index 7fbcc78..5782f7d 100644 --- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java @@ -17,6 +17,8 @@ package org.apache.ignite.yardstick.cache.load; +import java.io.BufferedReader; +import java.io.FileReader; import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; @@ -30,6 +32,10 @@ import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ThreadLocalRandom; +import javax.cache.configuration.FactoryBuilder; +import javax.cache.event.CacheEntryEvent; +import javax.cache.event.CacheEntryListenerException; +import javax.cache.event.CacheEntryUpdatedListener; import javax.cache.processor.EntryProcessor; import javax.cache.processor.EntryProcessorException; import javax.cache.processor.MutableEntry; @@ -40,12 +46,14 @@ import org.apache.ignite.IgniteCompute; import org.apache.ignite.IgniteDataStreamer; import org.apache.ignite.IgniteException; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheEntryEventSerializableFilter; import org.apache.ignite.cache.CacheMemoryMode; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CacheTypeMetadata; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.QueryIndex; import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cache.query.ContinuousQuery; import org.apache.ignite.cache.query.Query; import org.apache.ignite.cache.query.QueryCursor; import org.apache.ignite.cache.query.ScanQuery; @@ -62,6 +70,7 @@ import org.apache.ignite.transactions.TransactionIsolation; import org.apache.ignite.yardstick.IgniteAbstractBenchmark; import org.apache.ignite.yardstick.IgniteBenchmarkUtils; import org.apache.ignite.yardstick.cache.load.model.ModelUtil; +import org.jetbrains.annotations.NotNull; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.yardstickframework.BenchmarkConfiguration; @@ -94,9 +103,15 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark /** Map cache name on value classes. */ private Map valuesCacheClasses; - /** List of query descriptors by cache names */ + /** List of query descriptors by cache names. */ private Map> cacheSqlDescriptors; + /** List of SQL queries. */ + private List queries; + + /** List of allowed cache operations which will be executed. */ + private List allowedOperations; + /** * Replace value entry processor. */ @@ -107,6 +122,11 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark */ private BenchmarkRemoveEntryProcessor rmvEntryProc; + /** + * Last local continuous query cursor. + */ + private ThreadLocal localContinuousQueryCursor = new ThreadLocal<>(); + /** {@inheritDoc} */ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception { super.setUp(cfg); @@ -142,6 +162,10 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark rmvEntryProc = new BenchmarkRemoveEntryProcessor(); cacheSqlDescriptors = new HashMap<>(); + loadQueries(); + + loadAllowedOperations(); + for (String cacheName : ignite().cacheNames()) { IgniteCache cache = ignite().cache(cacheName); @@ -231,6 +255,45 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark } /** + * Load allowed operation from parameters. + */ + private void loadAllowedOperations() { + allowedOperations = new ArrayList<>(); + + if (args.allowOperations().isEmpty()) { + for (Operation op : Operation.values()) + allowedOperations.add(op); + } + else { + for (String opName : args.allowOperations()) + allowedOperations.add(Operation.valueOf(opName.toUpperCase())); + } + } + + /** + * Load query from file. + * + * @throws Exception If fail. + */ + private void loadQueries() throws Exception { + queries = new ArrayList<>(); + + if (args.queriesFile() != null) { + try (FileReader fr = new FileReader(args.queriesFile())) { + try (BufferedReader br = new BufferedReader(fr)) { + String line; + while ((line = br.readLine()) != null) { + if (line.trim().isEmpty()) + continue; + + queries.add(line.trim()); + } + } + } + } + } + + /** * @param cacheName Ignite cache name. * @param queryEntity Query entry. * @param valCls Class of value. @@ -418,7 +481,7 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark * @throws Exception If fail. */ private void executeRandomOperation(IgniteCache cache) throws Exception { - switch (Operation.valueOf(nextRandom(operations))) { + switch (nextRandomOperation()) { case PUT: doPut(cache); break; @@ -465,10 +528,22 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark case SQL_QUERY: doSqlQuery(cache); + break; + + case CONTINUOUS_QUERIE: + doContinuousQuery(cache); } } /** + * @return Operation. + */ + @NotNull private Operation nextRandomOperation() { + Operation op = allowedOperations.get(nextRandom(allowedOperations.size())); + return op; + } + + /** * Execute operations in transaction. * * @throws Exception if fail. @@ -557,7 +632,6 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark createRandomValue(i + 1, cache.getName())); else cache.invoke(createRandomKey(i, cache.getName()), rmvEntryProc); - } /** @@ -703,7 +777,11 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark int id = nextRandom(args.range()); - Query sq = nextBoolean() ? randomDescriptor.getSqlQuery(id) : randomDescriptor.getSqlFieldsQuery(id); + Query sq; + if (queries.isEmpty()) + sq = nextBoolean() ? randomDescriptor.getSqlQuery(id) : randomDescriptor.getSqlFieldsQuery(id); + else + sq = new SqlFieldsQuery(queries.get(nextRandom(queries.size()))); try (QueryCursor cursor = cache.query(sq)) { for (Object obj : cursor) @@ -713,6 +791,60 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark } /** + * @param cache Ignite cache. + * @throws Exception If failed. + */ + private void doContinuousQuery(IgniteCache cache) throws Exception { + if (nextBoolean() || localContinuousQueryCursor.get() == null) { + + if (localContinuousQueryCursor.get() != null) + localContinuousQueryCursor.get().close(); + + ContinuousQuery qry = new ContinuousQuery(); + + qry.setLocalListener(new ContinuousQueryUpdater()); + qry.setRemoteFilterFactory(FactoryBuilder.factoryOf(new ContinuousQueryFilter(nextRandom(100)))); + + localContinuousQueryCursor.set(cache.query(qry)); + } + } + + /** + * Continuous query updater class. + */ + private static class ContinuousQueryUpdater implements CacheEntryUpdatedListener, Serializable { + + /** {@inheritDoc} */ + @Override public void onUpdated(Iterable iterable) throws CacheEntryListenerException { + for (Object o : iterable); + } + } + + /** + * Continuous query filter class. + */ + private static class ContinuousQueryFilter implements CacheEntryEventSerializableFilter, Serializable { + + /** + * Value. + */ + private int val; + + /** + * @param val Value. + */ + public ContinuousQueryFilter(int val) { + this.val = val; + } + + /** {@inheritDoc} */ + @Override public boolean evaluate(CacheEntryEvent event) throws CacheEntryListenerException { + return event.getOldValue() != null && event.getValue() != null + && ((event.getOldValue().hashCode() - event.getValue().hashCode()) % (val + 3) == 0); + } + } + + /** * Closure for scan query executing. */ private static class ScanQueryBroadcastClosure implements IgniteRunnable { @@ -758,8 +890,7 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark scanQry.setFilter(igniteBiPred); try (QueryCursor cursor = cache.query(scanQry)) { - for (Object obj : cursor) - ; + for (Object obj : cursor); } } @@ -928,7 +1059,10 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark SCAN_QUERY, /** SQL query operation. */ - SQL_QUERY; + SQL_QUERY, + + /** Continuous Query. */ + CONTINUOUS_QUERIE; /** * @param num Number of operation. http://git-wip-us.apache.org/repos/asf/ignite/blob/339df384/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/ModelUtil.java ---------------------------------------------------------------------- diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/ModelUtil.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/ModelUtil.java index db6359e..fbaf67e 100644 --- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/ModelUtil.java +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/model/ModelUtil.java @@ -40,9 +40,9 @@ public class ModelUtil { private static Class[] keyClasses = { Double.class, Identifier.class, -// Mark.class, + Mark.class, Integer.class, -// UUID.class + UUID.class }; /**