ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From akuznet...@apache.org
Subject [14/14] incubator-ignite git commit: IGNITE-794 WIP on memory estimation.
Date Mon, 27 Apr 2015 11:02:51 GMT
IGNITE-794 WIP on memory estimation.


Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/5bcac1c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/5bcac1c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/5bcac1c7

Branch: refs/heads/ignite-794
Commit: 5bcac1c73f5528f10fffbf27353dce464df1568e
Parents: 779f341
Author: AKuznetsov <akuznetsov@gridgain.com>
Authored: Mon Apr 27 17:46:50 2015 +0700
Committer: AKuznetsov <akuznetsov@gridgain.com>
Committed: Mon Apr 27 17:46:50 2015 +0700

----------------------------------------------------------------------
 modules/core/pom.xml                            |   6 -
 .../ignite/internal/visor/cache/VisorCache.java |  81 ++++++---
 .../processors/query/h2/IgniteH2Indexing.java   |   9 +-
 modules/memory/pom.xml                          |  75 +++++++++
 .../apache/ignite/memory/MemoryEstimator.java   | 168 +++++++++++++++++++
 pom.xml                                         |   1 +
 6 files changed, 310 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/5bcac1c7/modules/core/pom.xml
----------------------------------------------------------------------
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index b374edf..62612f8 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -41,12 +41,6 @@
         </dependency>
 
         <dependency>
-            <groupId>org.ehcache</groupId>
-            <artifactId>sizeof</artifactId>
-            <version>0.3.0-SNAPSHOT</version>
-        </dependency>
-
-        <dependency>
             <groupId>mx4j</groupId>
             <artifactId>mx4j-tools</artifactId>
             <version>3.0.1</version>

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/5bcac1c7/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java
b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java
index 1ba003f..a2f24a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java
@@ -27,14 +27,13 @@ import org.apache.ignite.internal.processors.cache.*;
 import org.apache.ignite.internal.processors.cache.distributed.dht.*;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.*;
 import org.apache.ignite.internal.processors.cache.distributed.near.*;
-import org.apache.ignite.internal.processors.query.*;
 import org.apache.ignite.internal.util.lang.*;
 import org.apache.ignite.internal.util.typedef.internal.*;
 import org.apache.ignite.lang.*;
-import org.ehcache.sizeof.*;
 import org.jetbrains.annotations.*;
 
 import java.io.*;
+import java.lang.reflect.*;
 import java.util.*;
 import java.util.concurrent.*;
 
@@ -45,8 +44,17 @@ public class VisorCache implements Serializable {
     /** */
     private static final long serialVersionUID = 0L;
 
-    /** Default cache size sampling. */
-    private static final int DFLT_CACHE_SIZE_SAMPLING = 10;
+    /** */
+    private static final String MEM_ESTIMATOR = "MemoryEstimator";
+
+    /** */
+    private static final String MEM_ESTIMATOR_INSTALLED = "MemoryEstimatorInstalled";
+
+    /** */
+    private static final String MEM_ESTIMATOR_CACHE_SZ_MTHD = "MemoryEstimatorCacheSizeMethod";
+
+    /** */
+    private static final String MEM_ESTIMATOR_INDEXES_SZ_MTHD = "MemoryEstimatorIndexesSizeMethod";
 
     /** Cache name. */
     private String name;
@@ -60,6 +68,9 @@ public class VisorCache implements Serializable {
     /** Cache size in bytes. */
     private long memorySize;
 
+    /** Cache size in bytes. */
+    private long indexesSize;
+
     /** Number of all entries in cache. */
     private int size;
 
@@ -196,7 +207,6 @@ public class VisorCache implements Serializable {
         nearSize = ca.nearSize();
 
         dynamicDeploymentId = ca.context().dynamicDeploymentId();
-        memorySize = estimateMemorySize(ignite, ca, sample);
         dhtSize = size - nearSize;
         primarySize = ca.primarySize();
         offHeapAllocatedSize = ca.offHeapAllocatedSize();
@@ -204,6 +214,8 @@ public class VisorCache implements Serializable {
         partitions = ca.affinity().partitions();
         metrics = VisorCacheMetrics.from(ignite, ca);
 
+        estimateMemorySize(ignite, ca, sample);
+
         return this;
     }
 
@@ -216,34 +228,51 @@ public class VisorCache implements Serializable {
      * @return Estimation of memory size occupied by cache.
      * @throws IgniteCheckedException If estimation failed.
      */
-    protected long estimateMemorySize(IgniteEx ignite, GridCacheAdapter ca, int sample) throws
IgniteCheckedException {
-        ConcurrentMap<String, SizeOf> storage = ignite.cluster().nodeLocalMap();
+    protected void estimateMemorySize(IgniteEx ignite, GridCacheAdapter ca, int sample) throws
IgniteCheckedException {
+        ConcurrentMap<String, Object> storage = ignite.cluster().nodeLocalMap();
 
-        SizeOf sizeOf = storage.get("SizeOf");
+        Boolean installed = (Boolean)storage.get(MEM_ESTIMATOR_INSTALLED);
 
-        if (sizeOf == null)
-            sizeOf = SizeOf.newInstance();
+        if (installed == null) {
+            try {
+                Class<?> cls = Class.forName("org.apache.ignite.memory.MemoryEstimator");
 
-        int cnt = Math.min(sample > 0 ? sample : DFLT_CACHE_SIZE_SAMPLING, ca.size());
+                Object estimator = cls.newInstance();
 
-        long memSz = 0;
+                Method mtdCacheSz = cls.getMethod("estimateCacheSize", GridCacheAdapter.class,
int.class);
+                Method mtdIndexesSz = cls.getMethod("estimateIndexesSize", IgniteEx.class,
GridCacheAdapter.class);
 
-        if (cnt > 0) {
-            for (int i = 0; i < cnt; i++) {
-                GridCacheMapEntry entry = ca.randomInternalEntry();
+                memorySize = (Long)mtdCacheSz.invoke(estimator, ca, sample);
+                indexesSize = (Long)mtdIndexesSz.invoke(estimator, ignite, ca);
 
-                long ksz = sizeOf.deepSizeOf(Integer.MAX_VALUE, false, entry.key()).getCalculated();
-                long vsz = sizeOf.deepSizeOf(Integer.MAX_VALUE, false, entry.rawGet()).getCalculated();
-
-                memSz += (ksz + vsz);
+                storage.putIfAbsent(MEM_ESTIMATOR, estimator);
+                storage.putIfAbsent(MEM_ESTIMATOR_CACHE_SZ_MTHD, mtdCacheSz);
+                storage.putIfAbsent(MEM_ESTIMATOR_INDEXES_SZ_MTHD, mtdIndexesSz);
+                storage.putIfAbsent(MEM_ESTIMATOR_INSTALLED, Boolean.TRUE);
             }
+            catch (Exception e) {
+                storage.putIfAbsent(MEM_ESTIMATOR_INSTALLED, Boolean.FALSE);
 
-            memSz = (long)((double)memSz / cnt * size);
+                throw U.cast(e);
+            }
         }
+        else if (installed) {
+            Object estimator = storage.get(MEM_ESTIMATOR);
+            Method mtdCacheSz = (Method)storage.get(MEM_ESTIMATOR_CACHE_SZ_MTHD);
+            Method mtdIndexesSz = (Method)storage.get(MEM_ESTIMATOR_INDEXES_SZ_MTHD);
+
+            assert estimator != null && mtdCacheSz != null && mtdIndexesSz
!= null;
 
-        GridQueryIndexing indexing = ignite.context().query().indexing();
+            try {
+                memorySize = (Long)mtdCacheSz.invoke(estimator, ca, sample);
+                indexesSize = (Long)mtdIndexesSz.invoke(estimator, ignite, ca);
+            }
+            catch (Exception e) {
+                throw U.cast(e);
+            }
+        }
 
-        return memSz;
+        System.out.println("Cache: " + name + ", entries memory=" + memorySize + ", indexes
memory=" + indexesSize);
     }
 
     /**
@@ -255,6 +284,7 @@ public class VisorCache implements Serializable {
         c.name = name;
         c.mode = mode;
         c.memorySize = memorySize;
+        c.indexesSize = indexesSize;
         c.size = size;
         c.nearSize = nearSize;
         c.dhtSize = dhtSize;
@@ -300,6 +330,13 @@ public class VisorCache implements Serializable {
     }
 
     /**
+     * @return Indexes size in bytes.
+     */
+    public long indexesSize() {
+        return indexesSize;
+    }
+
+    /**
      * @return Number of all entries in cache.
      */
     public int size() {

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/5bcac1c7/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 1bc3409..4fcbe10 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -1044,7 +1044,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
      * @param schema Schema name.
      * @return Collection of table descriptors.
      */
-    private Collection<TableDescriptor> tables(String schema) {
+    public Collection<TableDescriptor> tables(String schema) {
         Schema s = schemas.get(schema);
 
         if (s == null)
@@ -1480,7 +1480,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
     /**
      * Information about table in database.
      */
-    private class TableDescriptor implements GridH2Table.IndexesFactory {
+    public class TableDescriptor implements GridH2Table.IndexesFactory {
         /** */
         private final String fullTblName;
 
@@ -1541,6 +1541,11 @@ public class IgniteH2Indexing implements GridQueryIndexing {
             return S.toString(TableDescriptor.class, this);
         }
 
+        public GridH2Table h2Table() {
+            return tbl;
+        }
+
+
         /** {@inheritDoc} */
         @Override public ArrayList<Index> createIndexes(GridH2Table tbl) {
             this.tbl = tbl;

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/5bcac1c7/modules/memory/pom.xml
----------------------------------------------------------------------
diff --git a/modules/memory/pom.xml b/modules/memory/pom.xml
new file mode 100644
index 0000000..4d373a1
--- /dev/null
+++ b/modules/memory/pom.xml
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<!--
+    POM file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.ignite</groupId>
+        <artifactId>ignite-parent</artifactId>
+        <version>1</version>
+        <relativePath>../../parent</relativePath>
+    </parent>
+
+    <artifactId>ignite-memory</artifactId>
+    <version>1.0.3-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-indexing</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.ehcache</groupId>
+            <artifactId>sizeof</artifactId>
+            <version>0.3.0-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/java</directory>
+                <excludes>
+                    <exclude>**/*.java</exclude>
+                </excludes>
+            </resource>
+        </resources>
+
+        <testResources>
+            <testResource>
+                <directory>src/test/java</directory>
+                <excludes>
+                    <exclude>**/*.java</exclude>
+                </excludes>
+            </testResource>
+        </testResources>
+    </build>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/5bcac1c7/modules/memory/src/main/java/org/apache/ignite/memory/MemoryEstimator.java
----------------------------------------------------------------------
diff --git a/modules/memory/src/main/java/org/apache/ignite/memory/MemoryEstimator.java b/modules/memory/src/main/java/org/apache/ignite/memory/MemoryEstimator.java
new file mode 100644
index 0000000..80c68c3
--- /dev/null
+++ b/modules/memory/src/main/java/org/apache/ignite/memory/MemoryEstimator.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.memory;
+
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.cache.*;
+import org.apache.ignite.internal.processors.query.*;
+import org.apache.ignite.internal.processors.query.h2.*;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.*;
+import org.apache.ignite.internal.processors.query.h2.opt.*;
+import org.ehcache.sizeof.*;
+import org.ehcache.sizeof.filters.*;
+import org.h2.index.*;
+
+import java.lang.reflect.*;
+import java.util.*;
+
+/**
+ * Utility class that will estimate memory occupied by cache.
+ */
+public class MemoryEstimator {
+    /** Default cache size sampling. */
+    private static final int DFLT_CACHE_SIZE_SAMPLING = 10;
+
+    /** */
+    private final SizeOf sizeOfCache;
+    /** */
+    private final SizeOf sizeOfIndex1;
+
+    private final Set<String> ignored = new HashSet<>();
+//    /** */
+//    private final SizeOf sizeOfIndex2;
+
+    /**
+     * Default constructor.
+     */
+    public MemoryEstimator() {
+        ignored.add("org.h2.engine.Database");
+        ignored.add("org.h2.message.TraceSystem");
+
+        SizeOfFilter filter1 = new SizeOfFilter() {
+            @Override public Collection<Field> filterFields(Class<?> klazz, Collection<Field>
fields) {
+                return fields;
+            }
+
+            @Override public boolean filterClass(Class<?> klazz) {
+                String klazzName = klazz.getName();
+
+                System.out.println("]]] " + klazzName);
+
+                return true;
+            }
+        };
+
+        SizeOfFilter filter = new SizeOfFilter() {
+            @Override public Collection<Field> filterFields(Class<?> klazz, Collection<Field>
fields) {
+                return fields;
+            }
+
+            @Override public boolean filterClass(Class<?> klazz) {
+                String klazzName = klazz.getName();
+
+                boolean f = (!klazzName.startsWith("org.apache.ignite") ||
+                    klazzName.contains("h2") || klazzName.contains("H2") ||
+                    klazzName.contains("snaptree")) &&
+                    !ignored.contains(klazzName);
+
+                System.out.println("))) " + klazzName + " " + f);
+
+                return f;
+            }
+        };
+
+        sizeOfCache = SizeOf.newInstance(filter1);
+        sizeOfIndex1 = SizeOf.newInstance(filter);
+    }
+
+    /**
+     * Estimate memory size for entries in cache.
+     *
+     * @param ca Cache to estimate.
+     * @param sample Number of entries to sample.
+     * @return Memory size in bytes under cache entries.
+     */
+    public long estimateCacheSize(GridCacheAdapter ca, int sample) {
+        int size = ca.size();
+
+        int cnt = Math.min(sample > 0 ? sample : DFLT_CACHE_SIZE_SAMPLING, size);
+
+        long memSz = 0;
+
+        if (cnt > 0) {
+            for (int i = 0; i < cnt; i++) {
+                GridCacheMapEntry entry = ca.randomInternalEntry();
+
+                long ksz = sizeOfCache.deepSizeOf(Integer.MAX_VALUE, false, entry.key()).getCalculated();
+                long vsz = sizeOfCache.deepSizeOf(Integer.MAX_VALUE, false, entry.rawGet()).getCalculated();
+
+                memSz += (ksz + vsz);
+            }
+
+            memSz = (long)((double)memSz / cnt * size);
+        }
+
+        return memSz;
+    }
+
+    /**
+     * Estimate memory size for cache indexes.
+     *
+     * @param ignite Ignite.
+     * @param ca Cache to estimate.
+     * @return Memory size in bytes under indexes.
+     */
+    public long estimateIndexesSize(IgniteEx ignite, GridCacheAdapter ca) {
+        long memSz = 0;
+
+        GridQueryIndexing indexing = ignite.context().query().indexing();
+
+        if (indexing instanceof IgniteH2Indexing) {
+            IgniteH2Indexing h2Indexing = (IgniteH2Indexing)indexing;
+
+            String cacheName = ca.name();
+
+            if (cacheName == null)
+                cacheName = "";
+
+            Collection<TableDescriptor> tbls = h2Indexing.tables(cacheName);
+
+            for (TableDescriptor tbl : tbls) {
+                GridH2Table h2Tbl = tbl.h2Table();
+
+                Iterator<Index> idxs = h2Tbl.getIndexes().iterator();
+
+                // Skip first index, as it is special wrapper.
+                if (idxs.hasNext())
+                    idxs.next();
+
+                while (idxs.hasNext()) {
+                    Index idx = idxs.next();
+
+                    long idxSz = sizeOfIndex1.deepSizeOf(Integer.MAX_VALUE, false, idx).getCalculated();
+
+                    System.out.println("Index=" + idx + ", size=" + idxSz);
+
+                    memSz += idxSz;
+                }
+            }
+        }
+
+        return memSz;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/5bcac1c7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8810b17..bc83d73 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,6 +74,7 @@
         <module>modules/slf4j</module>
         <module>modules/jcl</module>
         <module>modules/schema-import</module>
+        <module>modules/memory</module>
         <module>modules/codegen</module>
     </modules>
 


Mime
View raw message