ignite-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Zhengqingzheng <zhengqingzh...@huawei.com>
Subject re: Ignite cache data size problem.
Date Fri, 22 Apr 2016 12:38:00 GMT
Hi Vladimir,

My table is very simple, it contains the following information
OrgId (varchar) ,  oId(varchar),  fnum(int), gId(number), msg(varchar), num(number), date(date)
the gId is the primary Index.
--------------------------------------------------------------------------------------------------

And the java class is defined as:

package com.huawei.soa.ignite.test;

import java.io.Serializable;
import java.math.BigDecimal;
import java.util.Date;

import org.apache.ignite.cache.query.annotations.QuerySqlField;

public class UniqueField implements Serializable
{
    public String getOrgId()
    {
        return orgId;
    }

    public void setOrgId(String orgId)
    {
        this.orgId = orgId;
    }

    public String getOId()
    {
        return oId;
    }

    public void setOId(String oId)
    {
        this.oId = oId;
    }

    public String getGid()
    {
        return gId;
    }

    public void setGuid(String gId)
    {
        this.gId = gId;
    }

    public int getFNum()
    {
        return fNum;
    }

    public void setFNum(int fNum)
    {
        this.fNum = fNum;
    }

    public String getMsg()
    {
        return msg;
    }

    public void setMsg(String msg)
    {
        this.msg = msg;
    }

    public BigDecimal getNum()
    {
        return num;
    }

    public void setNum(BigDecimal num)
    {
        this.num = num;
    }

    public Date getDate()
    {
        return date;
    }

    public void setDate(Date date)
    {
        this.date = date;
    }

    @QuerySqlField
    private String orgId;

    @QuerySqlField(index=true)
    private String oId;

    @QuerySqlField(index=true)
    private String gId;

    @QuerySqlField
    private int fNum;

    @QuerySqlField
    private String msg;

    @QuerySqlField
    private BigDecimal num;

    @QuerySqlField
    private Date date;

    public UniqueField(){};

    public UniqueField(
            String orgId,
            String oId,
            String gId,
            int fNum,
            String msg,
            BigDecimal num,
            Date date
           ){
        this.orgId=orgId;
        this.oId=oId;
        this.gId = gId;
        this.fNum = fNum;
        this.msg = msg;
        this.num = num;
        this.date = date;
    }
}

--------------------------------------------------------------------------------------------------
My configuration file on the server side is listed as follows:

<?xml version="1.0" encoding="UTF-8"?>

<!--
  Licensed to the Apache Software Foundation (ASF) under one or more
  contributor license agreements.  See the NOTICE file distributed with
  this work for additional information regarding copyright ownership.
  The ASF licenses this file to You under the Apache License, Version 2.0
  (the "License"); you may not use this file except in compliance with
  the License.  You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
-->

<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xsi:schemaLocation="
       http://www.springframework.org/schema/beans
       http://www.springframework.org/schema/beans/spring-beans.xsd">
   <bean id="dataSource" class="oracle.jdbc.pool.OracleDataSource">
   <!-- real data is not listed -->
         <property name="URL" value="jdbc:oracle:thin:@xxx.xxx.xxx.xxx:xxx"   />
         <property name="user" value="xxx"/>
         <property name="password" value="xxx" />

   </bean>


<!--
        Alter configuration below as needed.
    -->


    <bean id="grid1.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
         <property name="peerClassLoadingEnabled" value="true" />
         <property name="cacheConfiguration">
         <list>
                   <bean class="org.apache.ignite.configuration.CacheConfiguration">

                            <property name="cacheStoreFactory">

                                     <!--  CacheJdbcPojoStoreExampleFactory is  extend
from CacheJdbcPojoStoreFactory -->
                                     <bean class="com.huawei.soa.ignite.util.CacheJdbcPojoStoreExampleFactory">
                                               <property name="dataSourceBean" value="dataSource"
/>
                                               <property name="dialect" >
                                                        <bean class="org.apache.ignite.cache.store.jdbc.dialect.OracleDialect"/>
                                               </property>
                                     </bean>
                            </property>

                            <!-- defintion from ignite document -->
                            <property name="memoryMode" value="OFFHEAP_TIERED" />
                            <!-- 10g offheap allocated-->
                            <property name="offHeapMaxMemory" value="#{10*1024L*1024L*1024L}"
/>
                            <property name="readThrough" value="true"  />
                            <property name="writeThrough" value="true" />
                            <property name="startSize" value="#{200*1024*1024}" />
                            <property name="cacheMode" value="PARTITIONED" />
                            <property name="backups" value="0" />

                   </bean>


         </list>
         </property>
         <property name="discoverySpi">
                   <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
                            <property name="ipFinder" >
                                     <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
                                               <property name="addresses">
                                                        <list>
                                                                 <value> xxx.xxx.xxx.xxx:47500..47509</value>
                                                                 <value> xxx.xxx.xxx.xxx:47500..47509</value>
                                                                 <value> xxx.xxx.xxx.xxx:47500..47509
</value>
                                                                 <value> xxx.xxx.xxx.xxx:47500..47509
</value>
                                                        </list>
                                               </property>
                                     </bean>
                            </property>
                   </bean>
         </property>
    </bean>

</beans>


Kind regards,
Kevin

发件人: Vladimir Ozerov [mailto:vozerov@gridgain.com]
发送时间: 2016年4月22日 20:13
收件人: user@ignite.apache.org
主题: Re: Ignite cache data size problem.

Hi,

It looks like you have relatively small entries (somewhere about 60-70 bytes per key-value
pair). Ignite also has some intrinsic overhead, which could be more than actual data in this
case. However, I surely would not expect that it will not fit into 80GB.

Could you please share your key and value model classes and your XML configuration to investigate
it further?

Vladimir.

On Fri, Apr 22, 2016 at 2:02 PM, Zhengqingzheng <zhengqingzheng@huawei.com<mailto:zhengqingzheng@huawei.com>>
wrote:
Hi there,
I am trying to load a table with 47Million records, in which the data size is less than 3gb.
However, When I load into the memory ( two vm with 48+32 = 80gb), it is still crushed due
to not enough memory space? This problem occurred when I instantiated 6 + 4 nodes.
Why the cache model need so much space ( 3g vs 80g)?
Any idea to explain this issue?

Kind regards,
Kevin


Mime
View raw message