ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ag...@apache.org
Subject ignite git commit: IGNITE-8322 Yardstick benchmark preloading option
Date Wed, 13 Jun 2018 18:31:56 GMT
Repository: ignite
Updated Branches:
  refs/heads/master c20b4b69e -> baad8117d


IGNITE-8322 Yardstick benchmark preloading option

Signed-off-by: Andrey Gura <agura@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/baad8117
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/baad8117
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/baad8117

Branch: refs/heads/master
Commit: baad8117df5f4acbbd4979537e7dad4e3a0c9e1c
Parents: c20b4b6
Author: oleg-ostanin <oostanin@gridagin.com>
Authored: Wed Jun 13 21:28:19 2018 +0300
Committer: Andrey Gura <agura@apache.org>
Committed: Wed Jun 13 21:28:19 2018 +0300

----------------------------------------------------------------------
 .../yardstick/config/ignite-pr-base-config.xml  | 150 +++++++++++++++
 .../config/ignite-pr-localhost-config.xml       | 108 +++++++++++
 .../yardstick/IgniteBenchmarkArguments.java     |  26 +++
 .../cache/IgniteCacheAbstractBenchmark.java     |  54 +++++-
 .../apache/ignite/yardstick/cache/Loader.java   | 189 +++++++++++++++++++
 5 files changed, 523 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/baad8117/modules/yardstick/config/ignite-pr-base-config.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-pr-base-config.xml b/modules/yardstick/config/ignite-pr-base-config.xml
new file mode 100644
index 0000000..9d7cddf
--- /dev/null
+++ b/modules/yardstick/config/ignite-pr-base-config.xml
@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    Ignite Spring configuration file to startup grid.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+    <bean id="base-ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"
abstract="true">
+        <property name="peerClassLoadingEnabled" value="false"/>
+
+        <property name="metricsLogFrequency" value="5000"/>
+
+        <property name="cacheConfiguration">
+            <list>
+                <bean class="org.apache.ignite.configuration.CacheConfiguration">
+                    <property name="dataRegionName" value="ATOMIC_REGION"/>
+
+                    <property name="name" value="atomic"/>
+
+                    <property name="cacheMode" value="PARTITIONED"/>
+
+                    <property name="atomicityMode" value="ATOMIC"/>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration">
+                    <property name="dataRegionName" value="TRANSACTIONAL_REGION"/>
+
+                    <property name="name" value="tx"/>
+
+                    <property name="cacheMode" value="PARTITIONED"/>
+
+                    <property name="atomicityMode" value="TRANSACTIONAL"/>
+
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration">
+                    <property name="name" value="atomic-index"/>
+
+                    <property name="cacheMode" value="PARTITIONED"/>
+
+                    <property name="atomicityMode" value="ATOMIC"/>
+
+                    <property name="indexedTypes">
+                        <list>
+                            <value>java.lang.Integer</value>
+                            <value>org.apache.ignite.yardstick.cache.model.Person1</value>
+
+                            <value>java.lang.Integer</value>
+                            <value>org.apache.ignite.yardstick.cache.model.Person2</value>
+
+                            <value>java.lang.Integer</value>
+                            <value>org.apache.ignite.yardstick.cache.model.Person8</value>
+                        </list>
+                    </property>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration">
+                    <property name="name" value="atomic-index-with-eviction"/>
+
+                    <property name="onheapCacheEnabled" value="true"/>
+
+                    <property name="evictionPolicy">
+                        <bean class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
+                            <!-- default range (1000000) x (1 + default backups number
(1)) / default nodes number (4) -->
+                            <constructor-arg value="500000"/>
+                        </bean>
+                    </property>
+
+                    <property name="cacheMode" value="PARTITIONED"/>
+
+                    <property name="atomicityMode" value="ATOMIC"/>
+
+                    <property name="indexedTypes">
+                        <list>
+                            <value>java.lang.Integer</value>
+                            <value>org.apache.ignite.yardstick.cache.model.Person1</value>
+
+                            <value>java.lang.Integer</value>
+                            <value>org.apache.ignite.yardstick.cache.model.Person2</value>
+
+                            <value>java.lang.Integer</value>
+                            <value>org.apache.ignite.yardstick.cache.model.Person8</value>
+                        </list>
+                    </property>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration">
+                    <property name="name" value="query"/>
+
+                    <property name="cacheMode" value="PARTITIONED"/>
+
+                    <property name="atomicityMode" value="ATOMIC"/>
+
+                    <property name="indexedTypes">
+                        <list>
+                            <value>java.lang.Integer</value>
+                            <value>org.apache.ignite.yardstick.cache.model.Organization</value>
+
+                            <value>java.lang.Integer</value>
+                            <value>org.apache.ignite.yardstick.cache.model.Person</value>
+
+                            <value>java.lang.Integer</value>
+                            <value>org.apache.ignite.yardstick.cache.model.SampleValue</value>
+
+                            <value>java.lang.Integer</value>
+                            <value>java.lang.Integer</value>
+                        </list>
+                    </property>
+                </bean>
+
+                <bean class="org.apache.ignite.configuration.CacheConfiguration">
+                    <property name="name" value="compute"/>
+
+                    <property name="cacheMode" value="PARTITIONED"/>
+
+                    <property name="atomicityMode" value="TRANSACTIONAL"/>
+                </bean>
+            </list>
+        </property>
+
+        <property name="includeEventTypes">
+            <list/>
+        </property>
+
+        <property name="loadBalancingSpi">
+            <bean class="org.apache.ignite.spi.loadbalancing.roundrobin.RoundRobinLoadBalancingSpi">
+                <property name="perTask" value="false"/>
+            </bean>
+        </property>
+    </bean>
+</beans>

http://git-wip-us.apache.org/repos/asf/ignite/blob/baad8117/modules/yardstick/config/ignite-pr-localhost-config.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-pr-localhost-config.xml b/modules/yardstick/config/ignite-pr-localhost-config.xml
new file mode 100644
index 0000000..8b8b838
--- /dev/null
+++ b/modules/yardstick/config/ignite-pr-localhost-config.xml
@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    Ignite Spring configuration file to startup grid.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+    <import resource="ignite-pr-base-config.xml"/>
+
+    <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration" parent="base-ignite.cfg">
+        <property name="localHost" value="127.0.0.1"/>
+
+        <property name="dataStorageConfiguration">
+            <bean class="org.apache.ignite.configuration.DataStorageConfiguration">
+                <property name="checkpointFrequency" value="60000"/>
+                <property name="walHistorySize" value="40"/>
+                <property name="writeThrottlingEnabled" value="true"/>
+
+                <!-- Defining several data regions for different memory regions -->
+                <property name="dataRegionConfigurations">
+                    <list>
+                        <!--
+                            Memory region of 40 MBs in size with an eviction enabled.
+                        -->
+                        <bean class="org.apache.ignite.configuration.DataRegionConfiguration">
+                            <property name="name" value="ATOMIC_REGION"/>
+                            <!-- Memory region of 40 MB initial size. -->
+                            <property name="initialSize" value="#{400 * 1024 * 1024}"/>
+                            <!-- Maximum size is 40 MB. -->
+                            <property name="maxSize" value="#{400 * 1024 * 1024}"/>
+                            <!-- Enabling eviction for this memory region -->
+                            <property name="pageEvictionMode" value="RANDOM_2_LRU"/>
+
+                            <property name="metricsEnabled" value="true"/>
+                            <property name="persistenceEnabled" value="true"/>
+                        </bean>
+
+                        <!--
+                            This memory region is backed by a memory-mapped file which names
is passed via
+                            'swapFilePath' parameter.
+                        -->
+                        <bean class="org.apache.ignite.configuration.DataRegionConfiguration">
+                            <property name="name" value="TRANSACTIONAL_REGION"/>
+                            <!-- Memory region of 40 MB initial size. -->
+                            <property name="initialSize" value="#{400 * 1024 * 1024}"/>
+                            <!-- Maximum size is 40 MB. -->
+                            <property name="maxSize" value="#{400 * 1024 * 1024}"/>
+                            <!-- Enabling eviction for this memory region -->
+                            <property name="pageEvictionMode" value="RANDOM_2_LRU"/>
+
+                            <property name="metricsEnabled" value="true"/>
+                            <property name="persistenceEnabled" value="true"/>
+                        </bean>
+                    </list>
+                </property>
+
+            </bean>
+        </property>
+
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+                        <property name="addresses">
+                            <list>
+                                <value>127.0.0.1:47500</value>
+                                <value>127.0.0.1:47501</value>
+                                <value>127.0.0.1:47502</value>
+                                <value>127.0.0.1:47503</value>
+                                <value>127.0.0.1:47504</value>
+                                <value>127.0.0.1:47505</value>
+                                <value>127.0.0.1:47506</value>
+                                <value>127.0.0.1:47507</value>
+                                <value>127.0.0.1:47508</value>
+                                <value>127.0.0.1:47509</value>
+                            </list>
+                        </property>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+
+        <property name="communicationSpi">
+            <bean class="org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi">
+                <property name="sharedMemoryPort" value="-1"/>
+            </bean>
+        </property>
+    </bean>
+</beans>

http://git-wip-us.apache.org/repos/asf/ignite/blob/baad8117/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
index 9789aea..e3f6d98 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
@@ -105,6 +105,14 @@ public class IgniteBenchmarkArguments {
     private int preloadAmount = 500_000;
 
     /** */
+    @Parameter(names = {"-pdrm", "--preloadDataRegionMult"}, description = "Data region size
multiplier for preload.")
+    private int preloadDataRegionMult = 0;
+
+    /** */
+    @Parameter(names = {"-ep", "--enablePreload"}, description = "Enable preload flag.")
+    private boolean enablePreload = true;
+
+    /** */
     @Parameter(names = {"-plfreq", "--preloadLogFrequency"}, description = "Interval between
printing logs")
     private long preloadLogsInterval = 30_000;
 
@@ -409,6 +417,10 @@ public class IgniteBenchmarkArguments {
         return range;
     }
 
+    public void setRange(int newVal) {
+        range = newVal;
+    }
+
     /**
      * @return Scale factor.
      */
@@ -424,6 +436,20 @@ public class IgniteBenchmarkArguments {
     }
 
     /**
+     * @return Preload data region multiplier.
+     */
+    public int preloadDataRegionMult() {
+        return preloadDataRegionMult;
+    }
+
+    /**
+     * @return Reset range for preload flag.
+     */
+    public boolean enablePreload() {
+        return enablePreload;
+    }
+
+    /**
      * @return Preload log printing interval in seconds.
      */
     public long preloadLogsInterval() {

http://git-wip-us.apache.org/repos/asf/ignite/blob/baad8117/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
index 28aa5a2..8b37dbe 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
@@ -29,7 +29,9 @@ import java.util.concurrent.Future;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteSemaphore;
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -105,6 +107,8 @@ public abstract class IgniteCacheAbstractBenchmark<K, V> extends
IgniteAbstractB
     @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
         super.setUp(cfg);
 
+        ignite().cluster().active(true);
+
         cache = cache();
 
         CacheConfiguration<?, ?> ccfg = cache.getConfiguration(CacheConfiguration.class);
@@ -112,8 +116,8 @@ public abstract class IgniteCacheAbstractBenchmark<K, V> extends
IgniteAbstractB
         String grpName = ccfg.getGroupName();
 
         BenchmarkUtils.println(cfg, "Benchmark setUp [name=" + getClass().getSimpleName()
+
-            ", cacheName="+ cache.getName() +
-            ", cacheGroup="+ grpName +
+            ", cacheName=" + cache.getName() +
+            ", cacheGroup=" + grpName +
             ", cacheCfg=" + cache.getConfiguration(CacheConfiguration.class) + ']');
 
         caches = args.cachesCount();
@@ -168,7 +172,7 @@ public abstract class IgniteCacheAbstractBenchmark<K, V> extends
IgniteAbstractB
                 }
             }
 
-            BenchmarkUtils.println(cfg, "Partition stats. [cacheName: "+ cache.getName()
+", topVer: "
+            BenchmarkUtils.println(cfg, "Partition stats. [cacheName: " + cache.getName()
+ ", topVer: "
                 + ignite().cluster().topologyVersion() + "]");
             BenchmarkUtils.println(cfg, "(Node id,  Number of Primary, Percent, Number of
Backup, Percent, Total, Percent)");
 
@@ -177,7 +181,7 @@ public abstract class IgniteCacheAbstractBenchmark<K, V> extends
IgniteAbstractB
                 List<Integer> backup = e.getValue().get2();
 
                 BenchmarkUtils.println(cfg, e.getKey().id() + "  "
-                    + primary.size() + "  " + primary.size() * 1. /aff.partitions() + " 
"
+                    + primary.size() + "  " + primary.size() * 1. / aff.partitions() + "
 "
                     + backup.size() + "  "
                     + backup.size() * 1. / (aff.partitions() * (args.backups() == 0 ? 1 :
args.backups())) + "  "
                     + (primary.size() + backup.size()) + "  "
@@ -185,6 +189,48 @@ public abstract class IgniteCacheAbstractBenchmark<K, V> extends
IgniteAbstractB
                 );
             }
         }
+
+        if(args.enablePreload()) {
+            startPreloadLogging(args.preloadLogsInterval());
+
+            preload();
+
+            stopPreloadLogging();
+        }
+    }
+
+    /**
+     * Preload data before benchmarking.
+     */
+    protected void preload() {
+        IgniteSemaphore semaphore = ignite().semaphore("preloadSemaphore",1,true,true);
+
+        semaphore.acquire();
+
+        try {
+            IgniteCache<String, Integer> preloadCache = ignite().getOrCreateCache("preloadCache");
+
+            if(preloadCache.get("loaded") == null) {
+                IgniteCompute compute = ignite().compute(ignite().cluster().forServers().forOldest());
+
+                IgniteCache<Integer, SampleValue> cache = (IgniteCache<Integer,
SampleValue>)cacheForOperation();
+
+                Integer res = compute.apply(new Loader(cache, args, ignite()), 0);
+
+                preloadCache.put("loaded", res);
+
+                if (res != null)
+                    args.setRange(res);
+            }
+            else {
+                BenchmarkUtils.println("Setting range to " + preloadCache.get("loaded"));
+
+                args.setRange(preloadCache.get("loaded"));
+            }
+        }
+        finally {
+            semaphore.release();
+        }
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/baad8117/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/Loader.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/Loader.java
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/Loader.java
new file mode 100644
index 0000000..ac8748d
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/Loader.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.yardstick.IgniteBenchmarkArguments;
+import org.apache.ignite.yardstick.cache.model.SampleValue;
+import org.jetbrains.annotations.NotNull;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Class for preload data before benchmarking PDS.
+ */
+public class Loader implements IgniteClosure<Integer, Integer> {
+    /** */
+    private AtomicBoolean loaded = new AtomicBoolean();
+
+    /** */
+    private IgniteCache<Integer, SampleValue> cache;
+
+    /** */
+    private IgniteBenchmarkArguments args;
+
+    /** */
+    private Ignite ignite;
+
+    /**
+     * Constructor.
+     *
+     * @param cache cache to preload data.
+     * @param args arguments.
+     * @param ignite Ignite instance.
+     */
+    Loader(IgniteCache<Integer, SampleValue> cache, IgniteBenchmarkArguments args,
Ignite ignite) {
+        this.cache = cache;
+        this.args = args;
+        this.ignite = ignite;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Integer apply(Integer integer) {
+        CacheConfiguration<Integer, SampleValue> cc = cache.getConfiguration(CacheConfiguration.class);
+
+        String dataRegName = cc.getDataRegionName();
+
+        BenchmarkUtils.println("Data region name = " + dataRegName);
+
+        DataStorageConfiguration dataStorCfg = ignite.configuration().getDataStorageConfiguration();
+
+        int pageSize = dataStorCfg.getPageSize();
+
+        BenchmarkUtils.println("Page size = " + pageSize);
+
+        DataRegionConfiguration dataRegCfg = null;
+
+        DataRegionConfiguration[] arr = ignite.configuration().getDataStorageConfiguration()
+            .getDataRegionConfigurations();
+
+        for (DataRegionConfiguration cfg : arr) {
+            if (cfg.getName().equals(dataRegName))
+                dataRegCfg = cfg;
+        }
+
+        if (dataRegCfg == null) {
+            BenchmarkUtils.println(String.format("Failed to get data region configuration
for cache %s",
+                cache.getName()));
+
+            return null;
+        }
+
+        long maxSize = dataRegCfg.getMaxSize();
+
+        BenchmarkUtils.println("Max size = " + maxSize);
+
+        long initSize = dataRegCfg.getInitialSize();
+
+        if (maxSize != initSize)
+            BenchmarkUtils.println("Initial data region size must be equal to max size!");
+
+        long pageNum = maxSize / pageSize;
+
+        BenchmarkUtils.println("Pages in data region: " + pageNum);
+
+        int cnt = 0;
+
+        final long pagesToLoad = pageNum * args.preloadDataRegionMult();
+
+        IgniteEx igniteEx = (IgniteEx)ignite;
+
+        try {
+            final DataRegionMetricsImpl impl = igniteEx.context().cache().context().database().dataRegion(dataRegName)
+                .memoryMetrics();
+
+            impl.enableMetrics();
+
+            BenchmarkUtils.println("Initial allocated pages = " + impl.getTotalAllocatedPages());
+
+            ExecutorService serv = Executors.newSingleThreadExecutor(new ThreadFactory()
{
+                @Override public Thread newThread(@NotNull Runnable r) {
+                    return new Thread(r, "Preload checker");
+                }
+            });
+
+            Future<?> checkFut = serv.submit(new Runnable() {
+                @Override public void run()  {
+                    while (!loaded.get()) {
+                        if (impl.getTotalAllocatedPages() >= pagesToLoad)
+                            loaded.getAndSet(true);
+
+                        try {
+                            Thread.sleep(500L);
+                        }
+                        catch (InterruptedException e) {
+                            BenchmarkUtils.error("Was interrupted while waiting before next
check.", e);
+                        }
+                    }
+                }
+            });
+
+            try (IgniteDataStreamer<Object, Object> streamer = ignite.dataStreamer(cache.getName()))
{
+                while (!loaded.get()) {
+                    streamer.addData(cnt++, new SampleValue());
+
+                    if (cnt % 1000_000 == 0) {
+                        long allocPages = impl.getTotalAllocatedPages();
+
+                        BenchmarkUtils.println("Load count = " + cnt);
+
+                        BenchmarkUtils.println("Allocated pages = " + allocPages);
+                    }
+                }
+            }
+            catch (Exception e){
+                BenchmarkUtils.error("Failed to load data.", e);
+            }
+
+            try {
+                checkFut.get();
+            }
+            catch (InterruptedException | ExecutionException e) {
+                BenchmarkUtils.error("Failed to check loading.", e);
+            }
+            finally {
+                serv.shutdown();
+            }
+
+            impl.disableMetrics();
+
+            BenchmarkUtils.println("Objects loaded = " + cnt);
+
+            BenchmarkUtils.println("Total allocated pages = " + impl.getTotalAllocatedPages());
+        }
+        catch (IgniteCheckedException e) {
+            BenchmarkUtils.error("Failed to load data.", e);
+        }
+
+        return cnt;
+    }
+}
\ No newline at end of file


Mime
View raw message