ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sboi...@apache.org
Subject [13/18] ignite git commit: IGNITE-3292 Fixed "Yardstick: add logging of preloading progress". This closes #1317.
Date Thu, 15 Dec 2016 09:21:19 GMT
IGNITE-3292 Fixed "Yardstick: add logging of preloading progress". This closes #1317.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7094c0fd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7094c0fd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7094c0fd

Branch: refs/heads/ignite-4371
Commit: 7094c0fd8dcf80ee17ca84e630565d11407f9c2c
Parents: cb1fd82
Author: oleg-ostanin <oostanin@gridgain.com>
Authored: Wed Dec 14 15:20:51 2016 +0300
Committer: nikolay_tikhonov <ntikhonov@gridgain.com>
Committed: Wed Dec 14 15:21:56 2016 +0300

----------------------------------------------------------------------
 .../yardstick/IgniteAbstractBenchmark.java      |  30 ++++
 .../yardstick/IgniteBenchmarkArguments.java     |  11 ++
 .../ignite/yardstick/IgniteBenchmarkUtils.java  |  42 ++++-
 .../apache/ignite/yardstick/PreloadLogger.java  | 155 +++++++++++++++++++
 .../IgniteCacheRandomOperationBenchmark.java    |  25 +--
 5 files changed, 253 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/7094c0fd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
index fa93f00..522499a 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
@@ -39,6 +39,9 @@ public abstract class IgniteAbstractBenchmark extends BenchmarkDriverAdapter
{
     /** Arguments. */
     protected final IgniteBenchmarkArguments args = new IgniteBenchmarkArguments();
 
+    /** Logger */
+    private PreloadLogger lgr;
+
     /** Node. */
     private IgniteNode node;
 
@@ -60,6 +63,33 @@ public abstract class IgniteAbstractBenchmark extends BenchmarkDriverAdapter
{
         waitForNodes();
     }
 
+    /**
+     * Prints non-system caches sizes during preload.
+     *
+     * @param logInterval time interval between printing preload log. Required to be positive.
+     */
+    protected void startPreloadLogging(long logInterval) {
+        try {
+            if (node != null && cfg != null && logInterval >= 0)
+                lgr = IgniteBenchmarkUtils.startPreloadLogger(node, cfg, logInterval);
+            else
+                BenchmarkUtils.println("Failed to start preload logger [node=" + node + ",
cfg = " + cfg +
+                    ", logInterval = " + logInterval + "]");
+        }
+        catch (Exception e) {
+            BenchmarkUtils.error("Failed to start preload logger [node=" + node + ", cfg
= " + cfg +
+                ", logInterval = " + logInterval + "]", e);
+        }
+    }
+
+    /**
+     * Terminates printing preload log.
+     */
+    protected void stopPreloadLogging() {
+        if (lgr != null)
+            lgr.stopAndPrintStatistics();
+    }
+
     /** {@inheritDoc} */
     @Override public void tearDown() throws Exception {
         if (node != null)

http://git-wip-us.apache.org/repos/asf/ignite/blob/7094c0fd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
index 1854938..2d2da5a 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
@@ -101,6 +101,10 @@ public class IgniteBenchmarkArguments {
     public int preloadAmount = 500_000;
 
     /** */
+    @Parameter(names = {"-plfreq", "--preloadLogFrequency"}, description = "Interval between
printing logs")
+    public long preloadLogsInterval = 30_000;
+
+    /** */
     @Parameter(names = {"-j", "--jobs"}, description = "Number of jobs for compute benchmarks")
     private int jobs = 10;
 
@@ -290,6 +294,13 @@ public class IgniteBenchmarkArguments {
     }
 
     /**
+     * @return Preload log printing interval in seconds.
+     */
+     public long preloadLogsInterval() {
+         return preloadLogsInterval;
+     }
+
+    /**
      * @return Configuration file.
      */
     public String configuration() {

http://git-wip-us.apache.org/repos/asf/ignite/blob/7094c0fd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
index 07549d5..c86dadb 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
@@ -20,6 +20,11 @@ package org.apache.ignite.yardstick;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteTransactions;
 import org.apache.ignite.Ignition;
@@ -31,14 +36,30 @@ import org.apache.ignite.transactions.TransactionIsolation;
 import org.apache.ignite.transactions.TransactionOptimisticException;
 import org.apache.ignite.transactions.TransactionRollbackException;
 import org.apache.ignite.yardstick.cache.IgnitePutBenchmark;
+import org.yardstickframework.BenchmarkConfiguration;
 import org.yardstickframework.BenchmarkDriver;
 import org.yardstickframework.BenchmarkDriverStartUp;
+import org.yardstickframework.BenchmarkUtils;
 
 /**
  * Utils.
  */
 public class IgniteBenchmarkUtils {
     /**
+     * Scheduler executor.
+     */
+    private static final ScheduledExecutorService exec =
+        Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
+            @Override public Thread newThread(Runnable run) {
+                Thread thread = Executors.defaultThreadFactory().newThread(run);
+
+                thread.setDaemon(true);
+
+                return thread;
+            }
+        });
+
+    /**
      * Utility class constructor.
      */
     private IgniteBenchmarkUtils() {
@@ -53,7 +74,7 @@ public class IgniteBenchmarkUtils {
      * @throws Exception If failed.
      */
     public static <T> T doInTransaction(IgniteTransactions igniteTx, TransactionConcurrency
txConcurrency,
-        TransactionIsolation txIsolation,  Callable<T> clo) throws Exception {
+        TransactionIsolation txIsolation, Callable<T> clo) throws Exception {
         while (true) {
             try (Transaction tx = igniteTx.txStart(txConcurrency, txIsolation)) {
                 T res = clo.call();
@@ -141,4 +162,23 @@ public class IgniteBenchmarkUtils {
         args.add(arg);
         args.add(val.toString());
     }
+
+    /**
+     * Prints non-system cache sizes during preload.
+     *
+     * @param node Ignite node.
+     * @param cfg Benchmark configuration.
+     * @param logsInterval Time interval in milliseconds between printing logs.
+     */
+    public static PreloadLogger startPreloadLogger(IgniteNode node, BenchmarkConfiguration
cfg, long logsInterval) {
+        PreloadLogger lgr = new PreloadLogger(node, cfg);
+
+        ScheduledFuture<?> fut = exec.scheduleWithFixedDelay(lgr, 0L, logsInterval,
TimeUnit.MILLISECONDS);
+
+        lgr.setFuture(fut);
+
+        BenchmarkUtils.println(cfg, "Preload logger was started.");
+
+        return lgr;
+    }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/7094c0fd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/PreloadLogger.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/PreloadLogger.java
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/PreloadLogger.java
new file mode 100644
index 0000000..c14a1df
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/PreloadLogger.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ScheduledFuture;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Prints non-system caches size.
+ */
+public class PreloadLogger implements Runnable {
+    /** Benchmark configuration. */
+    private BenchmarkConfiguration cfg;
+
+    /** List of caches whose size to be printed during preload. */
+    private Collection<IgniteCache<Object, Object>> caches;
+
+    /** Map for keeping previous values to make sure all the caches are working correctly.
*/
+    private Map<String, Long> cntrs;
+
+    /** String template used in String.format() to make output readable. */
+    private String strFmt;
+
+    /** Future instance to stop print log. */
+    private ScheduledFuture<?> fut;
+
+    /**
+     * @param node Ignite node.
+     * @param cfg BenchmarkConfiguration.
+     */
+    public PreloadLogger(IgniteNode node, BenchmarkConfiguration cfg) {
+        this.caches = new ArrayList<>();
+        this.cntrs = new HashMap<>();
+        this.cfg = cfg;
+
+        init(node);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void run() {
+        printCachesStatistics();
+    }
+
+    /**
+     * Prints non-system cache sizes.
+     */
+    public synchronized void printCachesStatistics() {
+        for (IgniteCache<Object, Object> cache : caches) {
+            try {
+                printCacheStatistics(cache);
+            }
+            catch (Exception e) {
+                BenchmarkUtils.println(cfg, "Failed to print cache size [cache=" + cache.getName()
+                    + ", msg=" + e.getMessage() + "]");
+            }
+        }
+    }
+
+    /**
+     * Print cache size along with amount of recently loaded entries.
+     *
+     * @param cache Ignite cache.
+     */
+    private void printCacheStatistics(IgniteCache<Object, Object> cache) {
+        String cacheName = cache.getName();
+
+        long cacheSize = cache.sizeLong();
+
+        long recentlyLoaded = cacheSize - cntrs.get(cacheName);
+        String recLoaded = recentlyLoaded == 0 ? String.valueOf(recentlyLoaded) : "+" + recentlyLoaded;
+
+        BenchmarkUtils.println(cfg, String.format(strFmt, cacheName, cacheSize, recLoaded));
+
+        cntrs.put(cacheName, cacheSize);
+    }
+
+    /**
+     * Helper method for initializing the cache list and the counters map.
+     *
+     * @param node Ignite node.
+     */
+    private void init(IgniteNode node) {
+        int longestName = 0;
+
+        for (String cacheName : node.ignite().cacheNames()) {
+            IgniteCache<Object, Object> cache = node.ignite().cache(cacheName);
+
+            caches.add(cache);
+
+            // Set up an initial values to the map.
+            cntrs.put(cache.getName(), 0L);
+
+            // Find out the length of the longest cache name.
+            longestName = Math.max(cache.getName().length(), longestName);
+        }
+
+        // Should look like "Preload:%-20s%-8d\t(%s)"
+        strFmt = "Preload:%-" + (longestName + 4) + "s%-8d\t(%s)";
+    }
+
+    /**
+     * Set future.
+     */
+    public void setFuture(ScheduledFuture<?> fut) {
+        this.fut = fut;
+    }
+
+    /**
+     * Terminates printing log.
+     */
+    public void stopAndPrintStatistics() {
+        try {
+            if (fut != null) {
+                if (!fut.cancel(true)) {
+                    U.sleep(200);
+
+                    if (!fut.cancel(true)) {
+                        BenchmarkUtils.println(cfg, "Failed to cancel Preload logger.");
+
+                        return;
+                    }
+                }
+            }
+
+            printCachesStatistics();
+        }
+        catch (Exception e) {
+            BenchmarkUtils.error("Failed to stop Preload logger.", e);
+        }
+
+        BenchmarkUtils.println(cfg, "Preload logger was stopped.");
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/7094c0fd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java
index d37cdca..590b64f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java
@@ -240,7 +240,8 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
 
                                 configureCacheSqlDescriptor(cacheName, queryEntity, valCls);
                             }
-                        } catch (ClassNotFoundException e) {
+                        }
+                        catch (ClassNotFoundException e) {
                             BenchmarkUtils.println(e.getMessage());
                             BenchmarkUtils.println("This can be a BinaryObject. Ignoring
exception.");
 
@@ -274,7 +275,8 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
                                     throw new IgniteException("Class is unknown for the load
test. Make sure you " +
                                         "specified its full name [clsName=" + cacheTypeMetadata.getKeyType()
+ ']');
                             }
-                        } catch (ClassNotFoundException e) {
+                        }
+                        catch (ClassNotFoundException e) {
                             BenchmarkUtils.println(e.getMessage());
                             BenchmarkUtils.println("This can be a BinaryObject. Ignoring
exception.");
 
@@ -401,8 +403,10 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
      */
     private void preLoading() throws Exception {
         if (args.preloadAmount() > args.range())
-            throw new IllegalArgumentException("Preloading amount (\"-pa\", \"--preloadAmount\")
must by less then the" +
-                " range (\"-r\", \"--range\").");
+            throw new IllegalArgumentException("Preloading amount (\"-pa\", \"--preloadAmount\")
" +
+                "must by less then the range (\"-r\", \"--range\").");
+
+        startPreloadLogging(args.preloadLogsInterval());
 
         Thread[] threads = new Thread[availableCaches.size()];
 
@@ -423,6 +427,8 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
 
         for (Thread thread : threads)
             thread.join();
+
+        stopPreloadLogging();
     }
 
     /**
@@ -641,8 +647,8 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
      * @param map Parameters map.
      */
     private void updateStat(Map<Object, Object> map) {
-        for (Operation op: Operation.values())
-            for (String cacheName: ignite().cacheNames()) {
+        for (Operation op : Operation.values())
+            for (String cacheName : ignite().cacheNames()) {
                 String opCacheKey = op + "_" + cacheName;
 
                 Integer val = (Integer)map.get(opCacheKey);
@@ -658,6 +664,7 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
 
     /**
      * Execute operations in transaction.
+     *
      * @param map Parameters map.
      * @throws Exception if fail.
      */
@@ -929,7 +936,7 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
 
         Integer[] sub = new Integer[cnt];
 
-        for (int i = 0; i< cnt; i++)
+        for (int i = 0; i < cnt; i++)
             sub[i] = nextRandom(args.range());
 
         sql = String.format(sql, sub);
@@ -994,7 +1001,7 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
 
         /** {@inheritDoc} */
         @Override public boolean evaluate(CacheEntryEvent evt) throws CacheEntryListenerException
{
-            return flag =! flag;
+            return flag = !flag;
         }
     }
 
@@ -1230,7 +1237,7 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
         private final boolean distributedJoin;
 
         /**
-          * @param sql SQL.
+         * @param sql SQL.
          * @param distributedJoin Distributed join flag.
          */
         public TestQuery(String sql, boolean distributedJoin) {


Mime
View raw message