incubator-directmemory-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Maurizio Cucchiara <mcucchi...@apache.org>
Subject Re: svn commit: r1228898 - in /incubator/directmemory/trunk/examples/src: main/java/org/apache/directmemory/examples/solr/ test/java/org/apache/directmemory/examples/solr/ test/resources/solr/config/
Date Sun, 08 Jan 2012 17:57:39 GMT
Ciao Tommaso,
sorry if I bother you, I don't want become that kind of person, but I have
to tell you that recently we reach a sort of consensus for the code
formatting style http://goo.gl/qFd4n
Please could you set your ide up to use the maven code style?
TIA

Twitter     :http://www.twitter.com/m_cucchiara
G+          :https://plus.google.com/107903711540963855921
Linkedin    :http://www.linkedin.com/in/mauriziocucchiara

Maurizio Cucchiara


On 8 January 2012 18:16, <tommaso@apache.org> wrote:

> Author: tommaso
> Date: Sun Jan  8 17:16:00 2012
> New Revision: 1228898
>
> URL: http://svn.apache.org/viewvc?rev=1228898&view=rev
> Log:
> [DIRECTMEMORY-59] - fixed statistics issue and added unit testing for it
>
> Added:
>
>  incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java
> Modified:
>
>  incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java
>
>  incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java
>
>  incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml
>
> Modified:
> incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java
> URL:
> http://svn.apache.org/viewvc/incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java?rev=1228898&r1=1228897&r2=1228898&view=diff
>
> ==============================================================================
> ---
> incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java
> (original)
> +++
> incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java
> Sun Jan  8 17:16:00 2012
> @@ -1,5 +1,3 @@
> -package org.apache.directmemory.examples.solr;
> -
>  /*
>  * Licensed to the Apache Software Foundation (ASF) under one
>  * or more contributor license agreements.  See the NOTICE file
> @@ -18,8 +16,11 @@ package org.apache.directmemory.examples
>  * specific language governing permissions and limitations
>  * under the License.
>  */
> +package org.apache.directmemory.examples.solr;
>
>  import org.apache.directmemory.cache.Cache;
> +import org.apache.directmemory.cache.CacheService;
> +import org.apache.directmemory.measures.Monitor;
>  import org.apache.directmemory.measures.Ram;
>  import org.apache.solr.common.util.NamedList;
>  import org.apache.solr.common.util.SimpleOrderedMap;
> @@ -37,11 +38,9 @@ import java.util.concurrent.atomic.Atomi
>  * A {@link SolrCache} based on Apache DirectMemory
>  */
>  public class SolrOffHeapCache<K, V>
> -    implements SolrCache<K, V>
> -{
> +        implements SolrCache<K, V> {
>
> -    private static class CumulativeStats
> -    {
> +    private static class CumulativeStats {
>         AtomicLong lookups = new AtomicLong();
>
>         AtomicLong hits = new AtomicLong();
> @@ -74,32 +73,32 @@ public class SolrOffHeapCache<K, V>
>     private String description = "DM Cache";
>
>     @Override
> -    public Object init( Map args, Object persistence, CacheRegenerator
> regenerator )
> -    {
> -        Object buffers = args.get( "buffers" );
> -        String size = String.valueOf( args.get( "size" ) );
> -        Cache.init( buffers != null ? Integer.valueOf( String.valueOf(
> buffers ) ) : 1,
> -                    Ram.Mb( Double.valueOf( size ) / 512 ) );
> +    public Object init(Map args, Object persistence, CacheRegenerator
> regenerator) {
> +        Object buffers = args.get("buffers");
> +        String size = String.valueOf(args.get("size"));
> +        Integer capacity =
> Integer.parseInt(String.valueOf(args.get("initialSize")));
> +        Cache.init(buffers != null ?
> Integer.valueOf(String.valueOf(buffers)) : 1,
> +                Ram.Mb(Double.valueOf(size) / 512),
> +                Ram.Mb(Double.valueOf(capacity) / 512),
> +                CacheService.DEFAULT_CONCURRENCY_LEVEL);
>
>         state = State.CREATED;
>         this.regenerator = regenerator;
> -        name = (String) args.get( "name" );
> +        name = (String) args.get("name");
>         String str = size;
> -        final int limit = str == null ? 1024 : Integer.parseInt( str );
> -        str = (String) args.get( "initialSize" );
> -        final int initialSize = Math.min( str == null ? 1024 :
> Integer.parseInt( str ), limit );
> -        str = (String) args.get( "autowarmCount" );
> -        autowarmCount = str == null ? 0 : Integer.parseInt( str );
> +        final int limit = str == null ? 1024 : Integer.parseInt(str);
> +        str = (String) args.get("initialSize");
> +        final int initialSize = Math.min(str == null ? 1024 :
> Integer.parseInt(str), limit);
> +        str = (String) args.get("autowarmCount");
> +        autowarmCount = str == null ? 0 : Integer.parseInt(str);
>
>         description = "Solr OffHeap Cache(maxSize=" + limit + ",
> initialSize=" + initialSize;
> -        if ( autowarmCount > 0 )
> -        {
> +        if (autowarmCount > 0) {
>             description += ", autowarmCount=" + autowarmCount + ",
> regenerator=" + regenerator;
>         }
>         description += ')';
>
> -        if ( persistence == null )
> -        {
> +        if (persistence == null) {
>             // must be the first time a cache of this type is being created
>             persistence = new CumulativeStats();
>         }
> @@ -110,138 +109,140 @@ public class SolrOffHeapCache<K, V>
>     }
>
>     @Override
> -    public String name()
> -    {
> +    public String name() {
>         return name;
>     }
>
>     @Override
> -    public int size()
> -    {
> -        return Long.valueOf( Cache.entries() ).intValue();
> +    public int size() {
> +        return Long.valueOf(Cache.entries()).intValue();
>     }
>
>     @Override
> -    public V put( K key, V value )
> -    {
> -        return (V) Cache.put( String.valueOf( key ), value );
> +    public V put(K key, V value) {
> +        synchronized (this) {
> +            if (state == State.LIVE) {
> +                stats.inserts.incrementAndGet();
> +            }
> +
> +            inserts++;
> +            return (V) Cache.put(String.valueOf(key), value);
> +        }
>     }
>
>     @Override
> -    public V get( K key )
> -    {
> -        return (V) Cache.retrieve( String.valueOf( key ) );
> +    public V get(K key) {
> +        synchronized (this) {
> +            V val = (V) Cache.retrieve(String.valueOf(key));
> +            if (state == State.LIVE) {
> +                // only increment lookups and hits if we are live.
> +                lookups++;
> +                stats.lookups.incrementAndGet();
> +                if (val != null) {
> +                    hits++;
> +                    stats.hits.incrementAndGet();
> +                }
> +            }
> +            return val;
> +        }
>     }
>
>     @Override
> -    public void clear()
> -    {
> -        synchronized ( this )
> -        {
> +    public void clear() {
> +        synchronized (this) {
>             Cache.clear();
>         }
>     }
>
>     @Override
> -    public void setState( State state )
> -    {
> +    public void setState(State state) {
>         this.state = state;
>     }
>
>     @Override
> -    public State getState()
> -    {
> +    public State getState() {
>         return state;
>     }
>
>     @Override
> -    public void warm( SolrIndexSearcher searcher, SolrCache<K, V> old )
> -        throws IOException
> -    {
> +    public void warm(SolrIndexSearcher searcher, SolrCache<K, V> old)
> +            throws IOException {
>         // it looks like there is no point in warming an off heap item
>     }
>
>     @Override
> -    public void close()
> -    {
> +    public void close() {
> +        Cache.dump();
> +        Monitor.dump();
>     }
>
>     @Override
> -    public String getName()
> -    {
> +    public String getName() {
>         return SolrOffHeapCache.class.getName();
>     }
>
>     @Override
> -    public String getVersion()
> -    {
> +    public String getVersion() {
>         return SolrCore.version;
>     }
>
> -    public String getDescription()
> -    {
> +    public String getDescription() {
>         return description;
>     }
>
> -    public Category getCategory()
> -    {
> +    public Category getCategory() {
>         return Category.CACHE;
>     }
>
>     @Override
> -    public String getSourceId()
> -    {
> +    public String getSourceId() {
>         return null;
>     }
>
>     @Override
> -    public String getSource()
> -    {
> +    public String getSource() {
>         return null;
>     }
>
>     @Override
> -    public URL[] getDocs()
> -    {
> +    public URL[] getDocs() {
>         return new URL[0];
>     }
>
> -    public NamedList getStatistics()
> -    {
> +    public NamedList getStatistics() {
>         NamedList lst = new SimpleOrderedMap();
> -        synchronized ( this )
> -        {
> -            lst.add( "lookups", lookups );
> -            lst.add( "hits", hits );
> -            lst.add( "hitratio", calcHitRatio( lookups, hits ) );
> -            lst.add( "inserts", inserts );
> -            lst.add( "evictions", evictions );
> -            lst.add( "size", Cache.entries() );
> +        synchronized (this) {
> +            lst.add("lookups", lookups);
> +            lst.add("hits", hits);
> +            lst.add("hitratio", calcHitRatio(lookups, hits));
> +            lst.add("inserts", inserts);
> +            lst.add("evictions", evictions);
> +            lst.add("size", Cache.entries());
>         }
>
> -        lst.add( "warmupTime", warmupTime );
> +        lst.add("warmupTime", warmupTime);
>
>         long clookups = stats.lookups.get();
>         long chits = stats.hits.get();
> -        lst.add( "cumulative_lookups", clookups );
> -        lst.add( "cumulative_hits", chits );
> -        lst.add( "cumulative_hitratio", calcHitRatio( clookups, chits ) );
> -        lst.add( "cumulative_inserts", stats.inserts.get() );
> -        lst.add( "cumulative_evictions", stats.evictions.get() );
> +        lst.add("cumulative_lookups", clookups);
> +        lst.add("cumulative_hits", chits);
> +        lst.add("cumulative_hitratio", calcHitRatio(clookups, chits));
> +        lst.add("cumulative_inserts", stats.inserts.get());
> +        lst.add("cumulative_evictions", stats.evictions.get());
>
>         return lst;
>     }
>
> -    private Object calcHitRatio( long clookups, long chits )
> -    {
> -        int ones = (int) ( hits * 100 / lookups );
> -        int tenths = (int) ( hits * 1000 / lookups ) - ones * 10;
> -        return Integer.toString( ones ) + '.' + tenths;
> +    private Object calcHitRatio(long clookups, long chits) {
> +        if (lookups == 0) return "0.00";
> +        if (lookups == hits) return "1.00";
> +        int hundredths = (int) (hits * 100 / lookups);   // rounded down
> +        if (hundredths < 10) return "0.0" + hundredths;
> +        return "0." + hundredths;
>     }
>
>     @Override
> -    public String toString()
> -    {
> +    public String toString() {
>         return name + getStatistics().toString();
>     }
>  }
>
> Added:
> incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java
> URL:
> http://svn.apache.org/viewvc/incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java?rev=1228898&view=auto
>
> ==============================================================================
> ---
> incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java
> (added)
> +++
> incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java
> Sun Jan  8 17:16:00 2012
> @@ -0,0 +1,87 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements.  See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License.  You may obtain a copy of the License at
> + *
> + *     http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.directmemory.examples.solr;
> +
> +import org.apache.lucene.search.MatchAllDocsQuery;
> +import org.apache.lucene.search.Query;
> +import org.apache.lucene.search.Sort;
> +import org.apache.solr.common.util.NamedList;
> +import org.apache.solr.search.QueryResultKey;
> +import org.apache.solr.search.function.DocValues;
> +import org.junit.After;
> +import org.junit.Before;
> +import org.junit.Test;
> +
> +import java.util.ArrayList;
> +import java.util.HashMap;
> +import java.util.Map;
> +
> +import static org.junit.Assert.*;
> +
> +/**
> + * Testcase for {@link SolrOffHeapCache}
> + */
> +public class SolrOffHeapCacheTest {
> +
> +    private SolrOffHeapCache solrOffHeapCache;
> +
> +    @Before
> +    public void setUp() {
> +        solrOffHeapCache = new SolrOffHeapCache();
> +        Map<String, String> args = new HashMap<String, String>();
> +        args.put("size", "10000");
> +        args.put("initialSize", "1000");
> +        solrOffHeapCache.init(args, null, null);
> +    }
> +
> +    @After
> +    public void tearDown() {
> +        solrOffHeapCache.clear();
> +        solrOffHeapCache.close();
> +    }
> +
> +    @Test
> +    public void testStatisticsWhenCacheNotUsedYet() {
> +        try {
> +            NamedList stats = solrOffHeapCache.getStatistics();
> +            assertNotNull(stats);
> +            assertEquals(0l, stats.get("lookups"));
> +            assertEquals(0l, stats.get("evictions"));
> +            assertEquals(0l, stats.get("hits"));
> +            assertEquals(0l, stats.get("inserts"));
> +        } catch (Exception e) {
> +            fail(e.getLocalizedMessage());
> +        }
> +    }
> +
> +    @Test
> +    public void testPut() {
> +        try {
> +            QueryResultKey queryResultKey = new QueryResultKey(new
> MatchAllDocsQuery(), new ArrayList<Query>(), new Sort(), 1);
> +            DocValues docValues = new DocValues() {
> +                @Override
> +                public String toString(int doc) {
> +                    return doc + "asd";
> +                }
> +            };
> +            solrOffHeapCache.put(queryResultKey, docValues);
> +        } catch (Exception e) {
> +            e.printStackTrace();
> +            fail(e.getLocalizedMessage());
> +        }
> +    }
> +}
>
> Modified:
> incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java
> URL:
> http://svn.apache.org/viewvc/incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java?rev=1228898&r1=1228897&r2=1228898&view=diff
>
> ==============================================================================
> ---
> incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java
> (original)
> +++
> incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java
> Sun Jan  8 17:16:00 2012
> @@ -19,7 +19,8 @@ package org.apache.directmemory.examples
>  import org.apache.directmemory.cache.Cache;
>  import org.apache.solr.request.LocalSolrQueryRequest;
>  import org.apache.solr.util.TestHarness;
> -import org.junit.Before;
> +import org.junit.AfterClass;
> +import org.junit.BeforeClass;
>  import org.junit.Test;
>
>  import static org.junit.Assert.assertTrue;
> @@ -27,41 +28,39 @@ import static org.junit.Assert.fail;
>
>  /**
>  */
> -public class SolrOffHeapIntegrationTest
> -{
> -    private TestHarness h;
> -
> -    @Before
> -    public void setUp()
> -    {
> +public class SolrOffHeapIntegrationTest {
> +    private static TestHarness h;
> +
> +    @BeforeClass
> +    public static void setUp() {
>         String data = "target/data/expand";
> -        String config = this.getClass().getResource(
> "/solr/config/solrconfig.xml" ).getFile();
> -        String schema = this.getClass().getResource(
> "/solr/config/schema.xml" ).getFile();
> -        this.h = new TestHarness( data, config, schema );
> +        String config =
> SolrOffHeapCache.class.getResource("/solr/config/solrconfig.xml").getFile();
> +        String schema =
> SolrOffHeapCache.class.getResource("/solr/config/schema.xml").getFile();
> +        h = new TestHarness(data, config, schema);
> +    }
> +
> +    @AfterClass
> +    public static void tearDown() {
> +        Cache.clear();
>     }
>
>     @Test
> -    public void testSimpleQuery()
> -    {
> -        try
> -        {
> +    public void testSimpleQuery() {
> +        try {
>             // add a doc to Solr
> -            h.validateAddDoc( "id", "1", "text", "something is happening
> here" );
> +            h.validateAddDoc("id", "1", "text", "something is happening
> here");
>
>             // make the query
> -            LocalSolrQueryRequest request = h.getRequestFactory(
> "standard", 0, 100 ).makeRequest( "q", "something" );
> -            String response = h.query( "standard", request );
> -            assertTrue( response != null );
> -            assertTrue( !response.contains( "error" ) );
> +            LocalSolrQueryRequest request =
> h.getRequestFactory("standard", 0, 10).makeRequest("q", "something");
> +            String response = h.query("standard", request);
> +            assertTrue(response != null);
> +            assertTrue(!response.contains("error"));
>
>             // check the cache has been hit
> -            assertTrue( Cache.entries() > 0 );
> +            assertTrue(Cache.entries() > 0);
>
> -        }
> -        catch ( Throwable e )
> -        {
> -            e.printStackTrace();
> -            fail( e.getLocalizedMessage() );
> +        } catch (Throwable e) {
> +            fail(e.getLocalizedMessage());
>         }
>     }
>  }
>
> Modified:
> incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml
> URL:
> http://svn.apache.org/viewvc/incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml?rev=1228898&r1=1228897&r2=1228898&view=diff
>
> ==============================================================================
> ---
> incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml
> (original)
> +++
> incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml
> Sun Jan  8 17:16:00 2012
> @@ -66,7 +66,7 @@
>     cause a serious error to be logged if it can't be loaded. <lib
>     path="../a-jar-that-does-not-exist.jar" />
>   -->
> -
> +   <luceneMatchVersion>LUCENE_35</luceneMatchVersion>
>
>   <!--
>     Used to specify an alternate directory to hold all index data other
> @@ -360,8 +360,8 @@
>       for each document). Since Lucene internal document ids are
>       transient, this cache will not be autowarmed.
>     -->
> -    <queryResultCache
> class="org.apache.directmemory.examples.solr.SolrOffHeapCache" size="512"
> -                      initialSize="512" autowarmCount="0" buffers="10"/>
> +    <queryResultCache
> class="org.apache.directmemory.examples.solr.SolrOffHeapCache" size="10000"
> +                      initialSize="1000" autowarmCount="0" buffers="2"/>
>
>     <!--
>       If true, stored fields that are not requested will be loaded
>
>
>

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message