ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From akuznet...@apache.org
Subject [09/46] ignite git commit: Performance optimizations.
Date Tue, 17 Nov 2015 13:34:47 GMT
http://git-wip-us.apache.org/repos/asf/ignite/blob/a4848a70/modules/core/src/test/java/org/apache/ignite/util/GridConcurrentLinkedHashMapMultiThreadedSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridConcurrentLinkedHashMapMultiThreadedSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/util/GridConcurrentLinkedHashMapMultiThreadedSelfTest.java
index 5c6cfcf..9fe2690 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridConcurrentLinkedHashMapMultiThreadedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridConcurrentLinkedHashMapMultiThreadedSelfTest.java
@@ -146,40 +146,41 @@ public class GridConcurrentLinkedHashMapMultiThreadedSelfTest extends
GridCommon
     public void testEvictPerSegment() throws Exception {
         info(">>> Test grid concurrent linked hash map...");
 
-        final int maxSize = 1000;
+        int concurLvl = 64;
+        final int maxSize = concurLvl * 30;
+        int diff = (int)(maxSize * 0.1);
 
         ConcurrentLinkedHashMap<Integer, String> linkedMap = new ConcurrentLinkedHashMap<>(
-            32, 0.75f, 64, maxSize, PER_SEGMENT_Q);
+            32, 0.75f, concurLvl, maxSize, PER_SEGMENT_Q);
 
         int keyCnt = 1000000;
 
-        putMultiThreaded(linkedMap, 10, keyCnt, maxSize);
-
-        int diff = 10; // 1% of 1000.
+        Map<String, LinkedList<Integer>> map = putMultiThreaded(
+            linkedMap,
+            10,
+            keyCnt,
+            maxSize * 10); // Intentionally memorize more than maxSize since in this mode
LRU is not fair.
 
         assertTrue("Invalid map size: " + linkedMap.size(), U.safeAbs(maxSize - linkedMap.size())
<= diff);
         assertTrue("Invalid map sizex: " + linkedMap.sizex(), U.safeAbs(maxSize - linkedMap.sizex())
<= diff);
 
-//      TODO IGNITE-606 - Need to fix iterators for ConcurrentLinkedHashMap in perSegment
mode
-//        LinkedList<Integer> keys = new LinkedList<Integer>(linkedMap.keySet());
-//
-//        while (!keys.isEmpty()) {
-//            boolean found = false;
-//
-//            int key = keys.removeLast();
-//
-//            for (LinkedList<Integer> threadKeys : map.values()) {
-//                if (threadKeys.getLast() == key) {
-//                    threadKeys.removeLast();
-//
-//                    found = true;
-//
-//                    break;
-//                }
-//            }
-//
-//            assertTrue("Key was not found on the top of any thread: " + key, found);
-//        }
+        LinkedList<Integer> keys = new LinkedList<>(linkedMap.keySet());
+
+        while (!keys.isEmpty()) {
+            boolean found = false;
+
+            int key = keys.removeLast();
+
+            for (LinkedList<Integer> threadKeys : map.values()) {
+                if (threadKeys.contains(key)) {
+                    found = true;
+
+                    break;
+                }
+            }
+
+            assertTrue("Key was not found in any thread: " + key, found);
+        }
 
         int min = Integer.MAX_VALUE;
         int max = 0;
@@ -207,40 +208,41 @@ public class GridConcurrentLinkedHashMapMultiThreadedSelfTest extends
GridCommon
     public void testEvictPerSegmentOptimizedRemoves() throws Exception {
         info(">>> Test grid concurrent linked hash map...");
 
-        final int maxSize = 1000;
+        int concurLvl = 64;
+        final int maxSize = concurLvl * 30;
+        int diff = (int)(maxSize * 0.1);
 
         ConcurrentLinkedHashMap<Integer, String> linkedMap = new ConcurrentLinkedHashMap<>(
-            32, 0.75f, 64, maxSize, PER_SEGMENT_Q_OPTIMIZED_RMV);
+            32, 0.75f, concurLvl, maxSize, PER_SEGMENT_Q_OPTIMIZED_RMV);
 
         int keyCnt = 1000000;
 
-        putMultiThreaded(linkedMap, 10, keyCnt, maxSize);
-
-        int diff = 10; // 1% of 1000.
+        Map<String, LinkedList<Integer>> map = putMultiThreaded(
+            linkedMap,
+            10,
+            keyCnt,
+            maxSize * 10); // Intentionally memorize more than maxSize since in this mode
LRU is not fair.
 
         assertTrue("Invalid map size: " + linkedMap.size(), U.safeAbs(maxSize - linkedMap.size())
<= diff);
         assertTrue("Invalid map sizex: " + linkedMap.sizex(), U.safeAbs(maxSize - linkedMap.sizex())
<= diff);
 
-//      TODO IGNITE-606 - Need to fix iterators for ConcurrentLinkedHashMap in perSegment
mode
-//        LinkedList<Integer> keys = new LinkedList<Integer>(linkedMap.keySet());
-//
-//        while (!keys.isEmpty()) {
-//            boolean found = false;
-//
-//            int key = keys.removeLast();
-//
-//            for (LinkedList<Integer> threadKeys : map.values()) {
-//                if (threadKeys.getLast() == key) {
-//                    threadKeys.removeLast();
-//
-//                    found = true;
-//
-//                    break;
-//                }
-//            }
-//
-//            assertTrue("Key was not found on the top of any thread: " + key, found);
-//        }
+        LinkedList<Integer> keys = new LinkedList<>(linkedMap.keySet());
+
+        while (!keys.isEmpty()) {
+            boolean found = false;
+
+            int key = keys.removeLast();
+
+            for (LinkedList<Integer> threadKeys : map.values()) {
+                if (threadKeys.contains(key)) {
+                    found = true;
+
+                    break;
+                }
+            }
+
+            assertTrue("Key was not found in any thread: " + key, found);
+        }
 
         int min = Integer.MAX_VALUE;
         int max = 0;
@@ -558,4 +560,4 @@ public class GridConcurrentLinkedHashMapMultiThreadedSelfTest extends
GridCommon
 
         info(">>> put get remove test complete [duration = " + (System.currentTimeMillis()
- start) + ']');
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/a4848a70/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
index ead3a63..9c3389f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxBenchmark.java
@@ -19,13 +19,23 @@ package org.apache.ignite.yardstick.cache;
 
 import java.util.Map;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.yardstick.cache.model.SampleValue;
+import org.yardstickframework.BenchmarkConfiguration;
 
 /**
  * Ignite benchmark that performs transactional put operations.
  */
 public class IgnitePutTxBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object>
{
     /** {@inheritDoc} */
+    @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        if (!IgniteSystemProperties.getBoolean("SKIP_MAP_CHECK"))
+            ignite().compute().broadcast(new WaitMapExchangeFinishCallable());
+    }
+
+    /** {@inheritDoc} */
     @Override public boolean test(Map<Object, Object> ctx) throws Exception {
         int key = nextRandom(args.range());
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/a4848a70/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxPrimaryOnlyBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxPrimaryOnlyBenchmark.java
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxPrimaryOnlyBenchmark.java
new file mode 100644
index 0000000..21275eb
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxPrimaryOnlyBenchmark.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.Map;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.yardstick.cache.model.SampleValue;
+import org.yardstickframework.BenchmarkConfiguration;
+
+/**
+ * Ignite benchmark that performs transactional put operations skipping key if local node
is backup.
+ */
+public class IgnitePutTxPrimaryOnlyBenchmark extends IgniteCacheAbstractBenchmark {
+    /** {@inheritDoc} */
+    @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        if (!IgniteSystemProperties.getBoolean("SKIP_MAP_CHECK"))
+            ignite().compute().broadcast(new WaitMapExchangeFinishCallable());
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        int key;
+
+        Affinity<Object> aff = ignite().affinity("tx");
+        ClusterNode locNode = ignite().cluster().localNode();
+
+        for (;;) {
+            key = nextRandom(args.range());
+
+            // Exit only if primary.
+            if (aff.isPrimary(locNode, key))
+                break;
+        }
+
+        // Implicit transaction is used.
+        cache.put(key, new SampleValue(key));
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteCache<Integer, Object> cache() {
+        return ignite().cache("tx");
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/a4848a70/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxSkipLocalBackupBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxSkipLocalBackupBenchmark.java
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxSkipLocalBackupBenchmark.java
new file mode 100644
index 0000000..63934e6
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxSkipLocalBackupBenchmark.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.Map;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.yardstick.cache.model.SampleValue;
+import org.yardstickframework.BenchmarkConfiguration;
+
+/**
+ * Ignite benchmark that performs transactional put operations skipping key if local node
is backup.
+ */
+public class IgnitePutTxSkipLocalBackupBenchmark extends IgniteCacheAbstractBenchmark {
+    /** {@inheritDoc} */
+    @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+        super.setUp(cfg);
+
+        if (!IgniteSystemProperties.getBoolean("SKIP_MAP_CHECK"))
+            ignite().compute().broadcast(new WaitMapExchangeFinishCallable());
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+        int key;
+
+        Affinity<Object> aff = ignite().affinity("tx");
+        ClusterNode locNode = ignite().cluster().localNode();
+
+        for (;;) {
+            key = nextRandom(args.range());
+
+            // Skip key if local node is backup.
+            if (!aff.isBackup(locNode, key))
+                break;
+        }
+
+        // Implicit transaction is used.
+        cache.put(key, new SampleValue(key));
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteCache<Integer, Object> cache() {
+        return ignite().cache("tx");
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/a4848a70/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/WaitMapExchangeFinishCallable.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/WaitMapExchangeFinishCallable.java
b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/WaitMapExchangeFinishCallable.java
new file mode 100644
index 0000000..83c50bd
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/WaitMapExchangeFinishCallable.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ *
+ */
+public class WaitMapExchangeFinishCallable implements IgniteCallable<Void> {
+    /** */
+    @IgniteInstanceResource
+    private Ignite ignite;
+
+    /** {@inheritDoc} */
+    @Override public Void call() throws Exception {
+        Collection<IgniteInternalCache<?, ?>> cachesx = ((IgniteKernal)ignite).cachesx(null);
+
+        for (IgniteInternalCache<?, ?> cache : cachesx) {
+            try {
+                GridDhtPartitionTopology top = cache.context().isNear() ? cache.context().near().dht().topology()
:
+                    cache.context().dht().topology();
+
+                BenchmarkUtils.println("Validating cache: " + cache.name());
+
+                for (;;) {
+                    boolean success = true;
+
+                    if (top.topologyVersion().topologyVersion() == ignite.cluster().topologyVersion())
{
+                        for (Map.Entry<UUID, GridDhtPartitionMap> e : top.partitionMap(true).entrySet())
{
+                            for (Map.Entry<Integer, GridDhtPartitionState> p : e.getValue().entrySet())
{
+                                if (p.getValue() != GridDhtPartitionState.OWNING) {
+                                    BenchmarkUtils.println("Not owning partition [part="
+ p.getKey() +
+                                        ", state=" + p.getValue() + ']');
+
+                                    success = false;
+
+                                    break;
+                                }
+                            }
+
+                            if (!success)
+                                break;
+                        }
+                    }
+                    else {
+                        BenchmarkUtils.println("Topology version is different [cache=" +
top.topologyVersion() +
+                            ", cluster=" + ignite.cluster().topologyVersion() + ']');
+
+                        success = false;
+                    }
+
+                    if (!success)
+                        Thread.sleep(1000);
+                    else {
+                        BenchmarkUtils.println("Cache state is fine: " + cache.name());
+
+                        break;
+                    }
+                }
+            }
+            catch (RuntimeException e1) {
+                BenchmarkUtils.println("Ignored exception: " + e1);
+            }
+        }
+
+        return null;
+    }
+}


Mime
View raw message