incubator-directmemory-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Tommaso Teofili <tommaso.teof...@gmail.com>
Subject Re: svn commit: r1228898 - in /incubator/directmemory/trunk/examples/src: main/java/org/apache/directmemory/examples/solr/ test/java/org/apache/directmemory/examples/solr/ test/resources/solr/config/
Date Sun, 08 Jan 2012 18:03:19 GMT
Hi Maurizio,
sure, no problem, my fault for having missed that.
I'll fix it.
Tommaso

2012/1/8 Maurizio Cucchiara <mcucchiara@apache.org>

> Ciao Tommaso,
> sorry if I bother you, I don't want become that kind of person, but I have
> to tell you that recently we reach a sort of consensus for the code
> formatting style http://goo.gl/qFd4n
> Please could you set your ide up to use the maven code style?
> TIA
>
> Twitter     :http://www.twitter.com/m_cucchiara
> G+          :https://plus.google.com/107903711540963855921
> Linkedin    :http://www.linkedin.com/in/mauriziocucchiara
>
> Maurizio Cucchiara
>
>
> On 8 January 2012 18:16, <tommaso@apache.org> wrote:
>
> > Author: tommaso
> > Date: Sun Jan  8 17:16:00 2012
> > New Revision: 1228898
> >
> > URL: http://svn.apache.org/viewvc?rev=1228898&view=rev
> > Log:
> > [DIRECTMEMORY-59] - fixed statistics issue and added unit testing for it
> >
> > Added:
> >
> >
>  incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java
> > Modified:
> >
> >
>  incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java
> >
> >
>  incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java
> >
> >
>  incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml
> >
> > Modified:
> >
> incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java
> > URL:
> >
> http://svn.apache.org/viewvc/incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java?rev=1228898&r1=1228897&r2=1228898&view=diff
> >
> >
> ==============================================================================
> > ---
> >
> incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java
> > (original)
> > +++
> >
> incubator/directmemory/trunk/examples/src/main/java/org/apache/directmemory/examples/solr/SolrOffHeapCache.java
> > Sun Jan  8 17:16:00 2012
> > @@ -1,5 +1,3 @@
> > -package org.apache.directmemory.examples.solr;
> > -
> >  /*
> >  * Licensed to the Apache Software Foundation (ASF) under one
> >  * or more contributor license agreements.  See the NOTICE file
> > @@ -18,8 +16,11 @@ package org.apache.directmemory.examples
> >  * specific language governing permissions and limitations
> >  * under the License.
> >  */
> > +package org.apache.directmemory.examples.solr;
> >
> >  import org.apache.directmemory.cache.Cache;
> > +import org.apache.directmemory.cache.CacheService;
> > +import org.apache.directmemory.measures.Monitor;
> >  import org.apache.directmemory.measures.Ram;
> >  import org.apache.solr.common.util.NamedList;
> >  import org.apache.solr.common.util.SimpleOrderedMap;
> > @@ -37,11 +38,9 @@ import java.util.concurrent.atomic.Atomi
> >  * A {@link SolrCache} based on Apache DirectMemory
> >  */
> >  public class SolrOffHeapCache<K, V>
> > -    implements SolrCache<K, V>
> > -{
> > +        implements SolrCache<K, V> {
> >
> > -    private static class CumulativeStats
> > -    {
> > +    private static class CumulativeStats {
> >         AtomicLong lookups = new AtomicLong();
> >
> >         AtomicLong hits = new AtomicLong();
> > @@ -74,32 +73,32 @@ public class SolrOffHeapCache<K, V>
> >     private String description = "DM Cache";
> >
> >     @Override
> > -    public Object init( Map args, Object persistence, CacheRegenerator
> > regenerator )
> > -    {
> > -        Object buffers = args.get( "buffers" );
> > -        String size = String.valueOf( args.get( "size" ) );
> > -        Cache.init( buffers != null ? Integer.valueOf( String.valueOf(
> > buffers ) ) : 1,
> > -                    Ram.Mb( Double.valueOf( size ) / 512 ) );
> > +    public Object init(Map args, Object persistence, CacheRegenerator
> > regenerator) {
> > +        Object buffers = args.get("buffers");
> > +        String size = String.valueOf(args.get("size"));
> > +        Integer capacity =
> > Integer.parseInt(String.valueOf(args.get("initialSize")));
> > +        Cache.init(buffers != null ?
> > Integer.valueOf(String.valueOf(buffers)) : 1,
> > +                Ram.Mb(Double.valueOf(size) / 512),
> > +                Ram.Mb(Double.valueOf(capacity) / 512),
> > +                CacheService.DEFAULT_CONCURRENCY_LEVEL);
> >
> >         state = State.CREATED;
> >         this.regenerator = regenerator;
> > -        name = (String) args.get( "name" );
> > +        name = (String) args.get("name");
> >         String str = size;
> > -        final int limit = str == null ? 1024 : Integer.parseInt( str );
> > -        str = (String) args.get( "initialSize" );
> > -        final int initialSize = Math.min( str == null ? 1024 :
> > Integer.parseInt( str ), limit );
> > -        str = (String) args.get( "autowarmCount" );
> > -        autowarmCount = str == null ? 0 : Integer.parseInt( str );
> > +        final int limit = str == null ? 1024 : Integer.parseInt(str);
> > +        str = (String) args.get("initialSize");
> > +        final int initialSize = Math.min(str == null ? 1024 :
> > Integer.parseInt(str), limit);
> > +        str = (String) args.get("autowarmCount");
> > +        autowarmCount = str == null ? 0 : Integer.parseInt(str);
> >
> >         description = "Solr OffHeap Cache(maxSize=" + limit + ",
> > initialSize=" + initialSize;
> > -        if ( autowarmCount > 0 )
> > -        {
> > +        if (autowarmCount > 0) {
> >             description += ", autowarmCount=" + autowarmCount + ",
> > regenerator=" + regenerator;
> >         }
> >         description += ')';
> >
> > -        if ( persistence == null )
> > -        {
> > +        if (persistence == null) {
> >             // must be the first time a cache of this type is being
> created
> >             persistence = new CumulativeStats();
> >         }
> > @@ -110,138 +109,140 @@ public class SolrOffHeapCache<K, V>
> >     }
> >
> >     @Override
> > -    public String name()
> > -    {
> > +    public String name() {
> >         return name;
> >     }
> >
> >     @Override
> > -    public int size()
> > -    {
> > -        return Long.valueOf( Cache.entries() ).intValue();
> > +    public int size() {
> > +        return Long.valueOf(Cache.entries()).intValue();
> >     }
> >
> >     @Override
> > -    public V put( K key, V value )
> > -    {
> > -        return (V) Cache.put( String.valueOf( key ), value );
> > +    public V put(K key, V value) {
> > +        synchronized (this) {
> > +            if (state == State.LIVE) {
> > +                stats.inserts.incrementAndGet();
> > +            }
> > +
> > +            inserts++;
> > +            return (V) Cache.put(String.valueOf(key), value);
> > +        }
> >     }
> >
> >     @Override
> > -    public V get( K key )
> > -    {
> > -        return (V) Cache.retrieve( String.valueOf( key ) );
> > +    public V get(K key) {
> > +        synchronized (this) {
> > +            V val = (V) Cache.retrieve(String.valueOf(key));
> > +            if (state == State.LIVE) {
> > +                // only increment lookups and hits if we are live.
> > +                lookups++;
> > +                stats.lookups.incrementAndGet();
> > +                if (val != null) {
> > +                    hits++;
> > +                    stats.hits.incrementAndGet();
> > +                }
> > +            }
> > +            return val;
> > +        }
> >     }
> >
> >     @Override
> > -    public void clear()
> > -    {
> > -        synchronized ( this )
> > -        {
> > +    public void clear() {
> > +        synchronized (this) {
> >             Cache.clear();
> >         }
> >     }
> >
> >     @Override
> > -    public void setState( State state )
> > -    {
> > +    public void setState(State state) {
> >         this.state = state;
> >     }
> >
> >     @Override
> > -    public State getState()
> > -    {
> > +    public State getState() {
> >         return state;
> >     }
> >
> >     @Override
> > -    public void warm( SolrIndexSearcher searcher, SolrCache<K, V> old )
> > -        throws IOException
> > -    {
> > +    public void warm(SolrIndexSearcher searcher, SolrCache<K, V> old)
> > +            throws IOException {
> >         // it looks like there is no point in warming an off heap item
> >     }
> >
> >     @Override
> > -    public void close()
> > -    {
> > +    public void close() {
> > +        Cache.dump();
> > +        Monitor.dump();
> >     }
> >
> >     @Override
> > -    public String getName()
> > -    {
> > +    public String getName() {
> >         return SolrOffHeapCache.class.getName();
> >     }
> >
> >     @Override
> > -    public String getVersion()
> > -    {
> > +    public String getVersion() {
> >         return SolrCore.version;
> >     }
> >
> > -    public String getDescription()
> > -    {
> > +    public String getDescription() {
> >         return description;
> >     }
> >
> > -    public Category getCategory()
> > -    {
> > +    public Category getCategory() {
> >         return Category.CACHE;
> >     }
> >
> >     @Override
> > -    public String getSourceId()
> > -    {
> > +    public String getSourceId() {
> >         return null;
> >     }
> >
> >     @Override
> > -    public String getSource()
> > -    {
> > +    public String getSource() {
> >         return null;
> >     }
> >
> >     @Override
> > -    public URL[] getDocs()
> > -    {
> > +    public URL[] getDocs() {
> >         return new URL[0];
> >     }
> >
> > -    public NamedList getStatistics()
> > -    {
> > +    public NamedList getStatistics() {
> >         NamedList lst = new SimpleOrderedMap();
> > -        synchronized ( this )
> > -        {
> > -            lst.add( "lookups", lookups );
> > -            lst.add( "hits", hits );
> > -            lst.add( "hitratio", calcHitRatio( lookups, hits ) );
> > -            lst.add( "inserts", inserts );
> > -            lst.add( "evictions", evictions );
> > -            lst.add( "size", Cache.entries() );
> > +        synchronized (this) {
> > +            lst.add("lookups", lookups);
> > +            lst.add("hits", hits);
> > +            lst.add("hitratio", calcHitRatio(lookups, hits));
> > +            lst.add("inserts", inserts);
> > +            lst.add("evictions", evictions);
> > +            lst.add("size", Cache.entries());
> >         }
> >
> > -        lst.add( "warmupTime", warmupTime );
> > +        lst.add("warmupTime", warmupTime);
> >
> >         long clookups = stats.lookups.get();
> >         long chits = stats.hits.get();
> > -        lst.add( "cumulative_lookups", clookups );
> > -        lst.add( "cumulative_hits", chits );
> > -        lst.add( "cumulative_hitratio", calcHitRatio( clookups, chits )
> );
> > -        lst.add( "cumulative_inserts", stats.inserts.get() );
> > -        lst.add( "cumulative_evictions", stats.evictions.get() );
> > +        lst.add("cumulative_lookups", clookups);
> > +        lst.add("cumulative_hits", chits);
> > +        lst.add("cumulative_hitratio", calcHitRatio(clookups, chits));
> > +        lst.add("cumulative_inserts", stats.inserts.get());
> > +        lst.add("cumulative_evictions", stats.evictions.get());
> >
> >         return lst;
> >     }
> >
> > -    private Object calcHitRatio( long clookups, long chits )
> > -    {
> > -        int ones = (int) ( hits * 100 / lookups );
> > -        int tenths = (int) ( hits * 1000 / lookups ) - ones * 10;
> > -        return Integer.toString( ones ) + '.' + tenths;
> > +    private Object calcHitRatio(long clookups, long chits) {
> > +        if (lookups == 0) return "0.00";
> > +        if (lookups == hits) return "1.00";
> > +        int hundredths = (int) (hits * 100 / lookups);   // rounded down
> > +        if (hundredths < 10) return "0.0" + hundredths;
> > +        return "0." + hundredths;
> >     }
> >
> >     @Override
> > -    public String toString()
> > -    {
> > +    public String toString() {
> >         return name + getStatistics().toString();
> >     }
> >  }
> >
> > Added:
> >
> incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java
> > URL:
> >
> http://svn.apache.org/viewvc/incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java?rev=1228898&view=auto
> >
> >
> ==============================================================================
> > ---
> >
> incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java
> > (added)
> > +++
> >
> incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapCacheTest.java
> > Sun Jan  8 17:16:00 2012
> > @@ -0,0 +1,87 @@
> > +/**
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements.  See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License.  You may obtain a copy of the License at
> > + *
> > + *     http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> > implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.directmemory.examples.solr;
> > +
> > +import org.apache.lucene.search.MatchAllDocsQuery;
> > +import org.apache.lucene.search.Query;
> > +import org.apache.lucene.search.Sort;
> > +import org.apache.solr.common.util.NamedList;
> > +import org.apache.solr.search.QueryResultKey;
> > +import org.apache.solr.search.function.DocValues;
> > +import org.junit.After;
> > +import org.junit.Before;
> > +import org.junit.Test;
> > +
> > +import java.util.ArrayList;
> > +import java.util.HashMap;
> > +import java.util.Map;
> > +
> > +import static org.junit.Assert.*;
> > +
> > +/**
> > + * Testcase for {@link SolrOffHeapCache}
> > + */
> > +public class SolrOffHeapCacheTest {
> > +
> > +    private SolrOffHeapCache solrOffHeapCache;
> > +
> > +    @Before
> > +    public void setUp() {
> > +        solrOffHeapCache = new SolrOffHeapCache();
> > +        Map<String, String> args = new HashMap<String, String>();
> > +        args.put("size", "10000");
> > +        args.put("initialSize", "1000");
> > +        solrOffHeapCache.init(args, null, null);
> > +    }
> > +
> > +    @After
> > +    public void tearDown() {
> > +        solrOffHeapCache.clear();
> > +        solrOffHeapCache.close();
> > +    }
> > +
> > +    @Test
> > +    public void testStatisticsWhenCacheNotUsedYet() {
> > +        try {
> > +            NamedList stats = solrOffHeapCache.getStatistics();
> > +            assertNotNull(stats);
> > +            assertEquals(0l, stats.get("lookups"));
> > +            assertEquals(0l, stats.get("evictions"));
> > +            assertEquals(0l, stats.get("hits"));
> > +            assertEquals(0l, stats.get("inserts"));
> > +        } catch (Exception e) {
> > +            fail(e.getLocalizedMessage());
> > +        }
> > +    }
> > +
> > +    @Test
> > +    public void testPut() {
> > +        try {
> > +            QueryResultKey queryResultKey = new QueryResultKey(new
> > MatchAllDocsQuery(), new ArrayList<Query>(), new Sort(), 1);
> > +            DocValues docValues = new DocValues() {
> > +                @Override
> > +                public String toString(int doc) {
> > +                    return doc + "asd";
> > +                }
> > +            };
> > +            solrOffHeapCache.put(queryResultKey, docValues);
> > +        } catch (Exception e) {
> > +            e.printStackTrace();
> > +            fail(e.getLocalizedMessage());
> > +        }
> > +    }
> > +}
> >
> > Modified:
> >
> incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java
> > URL:
> >
> http://svn.apache.org/viewvc/incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java?rev=1228898&r1=1228897&r2=1228898&view=diff
> >
> >
> ==============================================================================
> > ---
> >
> incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java
> > (original)
> > +++
> >
> incubator/directmemory/trunk/examples/src/test/java/org/apache/directmemory/examples/solr/SolrOffHeapIntegrationTest.java
> > Sun Jan  8 17:16:00 2012
> > @@ -19,7 +19,8 @@ package org.apache.directmemory.examples
> >  import org.apache.directmemory.cache.Cache;
> >  import org.apache.solr.request.LocalSolrQueryRequest;
> >  import org.apache.solr.util.TestHarness;
> > -import org.junit.Before;
> > +import org.junit.AfterClass;
> > +import org.junit.BeforeClass;
> >  import org.junit.Test;
> >
> >  import static org.junit.Assert.assertTrue;
> > @@ -27,41 +28,39 @@ import static org.junit.Assert.fail;
> >
> >  /**
> >  */
> > -public class SolrOffHeapIntegrationTest
> > -{
> > -    private TestHarness h;
> > -
> > -    @Before
> > -    public void setUp()
> > -    {
> > +public class SolrOffHeapIntegrationTest {
> > +    private static TestHarness h;
> > +
> > +    @BeforeClass
> > +    public static void setUp() {
> >         String data = "target/data/expand";
> > -        String config = this.getClass().getResource(
> > "/solr/config/solrconfig.xml" ).getFile();
> > -        String schema = this.getClass().getResource(
> > "/solr/config/schema.xml" ).getFile();
> > -        this.h = new TestHarness( data, config, schema );
> > +        String config =
> >
> SolrOffHeapCache.class.getResource("/solr/config/solrconfig.xml").getFile();
> > +        String schema =
> > SolrOffHeapCache.class.getResource("/solr/config/schema.xml").getFile();
> > +        h = new TestHarness(data, config, schema);
> > +    }
> > +
> > +    @AfterClass
> > +    public static void tearDown() {
> > +        Cache.clear();
> >     }
> >
> >     @Test
> > -    public void testSimpleQuery()
> > -    {
> > -        try
> > -        {
> > +    public void testSimpleQuery() {
> > +        try {
> >             // add a doc to Solr
> > -            h.validateAddDoc( "id", "1", "text", "something is happening
> > here" );
> > +            h.validateAddDoc("id", "1", "text", "something is happening
> > here");
> >
> >             // make the query
> > -            LocalSolrQueryRequest request = h.getRequestFactory(
> > "standard", 0, 100 ).makeRequest( "q", "something" );
> > -            String response = h.query( "standard", request );
> > -            assertTrue( response != null );
> > -            assertTrue( !response.contains( "error" ) );
> > +            LocalSolrQueryRequest request =
> > h.getRequestFactory("standard", 0, 10).makeRequest("q", "something");
> > +            String response = h.query("standard", request);
> > +            assertTrue(response != null);
> > +            assertTrue(!response.contains("error"));
> >
> >             // check the cache has been hit
> > -            assertTrue( Cache.entries() > 0 );
> > +            assertTrue(Cache.entries() > 0);
> >
> > -        }
> > -        catch ( Throwable e )
> > -        {
> > -            e.printStackTrace();
> > -            fail( e.getLocalizedMessage() );
> > +        } catch (Throwable e) {
> > +            fail(e.getLocalizedMessage());
> >         }
> >     }
> >  }
> >
> > Modified:
> >
> incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml
> > URL:
> >
> http://svn.apache.org/viewvc/incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml?rev=1228898&r1=1228897&r2=1228898&view=diff
> >
> >
> ==============================================================================
> > ---
> >
> incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml
> > (original)
> > +++
> >
> incubator/directmemory/trunk/examples/src/test/resources/solr/config/solrconfig.xml
> > Sun Jan  8 17:16:00 2012
> > @@ -66,7 +66,7 @@
> >     cause a serious error to be logged if it can't be loaded. <lib
> >     path="../a-jar-that-does-not-exist.jar" />
> >   -->
> > -
> > +   <luceneMatchVersion>LUCENE_35</luceneMatchVersion>
> >
> >   <!--
> >     Used to specify an alternate directory to hold all index data other
> > @@ -360,8 +360,8 @@
> >       for each document). Since Lucene internal document ids are
> >       transient, this cache will not be autowarmed.
> >     -->
> > -    <queryResultCache
> > class="org.apache.directmemory.examples.solr.SolrOffHeapCache" size="512"
> > -                      initialSize="512" autowarmCount="0" buffers="10"/>
> > +    <queryResultCache
> > class="org.apache.directmemory.examples.solr.SolrOffHeapCache"
> size="10000"
> > +                      initialSize="1000" autowarmCount="0" buffers="2"/>
> >
> >     <!--
> >       If true, stored fields that are not requested will be loaded
> >
> >
> >
>

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message