maven-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ogusa...@apache.org
Subject svn commit: r776133 - in /maven/mercury/trunk: mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/ mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/ mercury-core/src/main/java/org/apache/maven/mercury/util/ me...
Date Mon, 18 May 2009 23:17:24 GMT
Author: ogusakov
Date: Mon May 18 23:17:24 2009
New Revision: 776133

URL: http://svn.apache.org/viewvc?rev=776133&view=rev
Log:
[MERCURY-123] - trying to cache results from DependencyProcessor() to enable once-only processing
on POMs. This should be linked to session-like behavior because next session may happen in
quite some time and data should be properly aged

Added:
    maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
  (with props)
    maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
  (with props)
Modified:
    maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java
    maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java
    maven/mercury/trunk/mercury-it/pom.xml
    maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom
  (props changed)
    maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1

Modified: maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java
URL: http://svn.apache.org/viewvc/maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java?rev=776133&r1=776132&r2=776133&view=diff
==============================================================================
--- maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java
(original)
+++ maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java
Mon May 18 23:17:24 2009
@@ -62,6 +62,16 @@
 
     public static final String EVENT_SAVE_RAW = "save.raw";
 
+//    public static final String SYSTEM_PROPERTY_CACHE_METADATA = "mercury.cache.metadata";
+//    
+//    /** by default - cache metadata in memory */
+//    private static final boolean cacheMetadata = Boolean.valueOf( System.getProperty( SYSTEM_PROPERTY_CACHE_METADATA,
"true" ) );
+
+    public static final String SYSTEM_PROPERTY_CACHE_RAW = "mercury.cache.raw";
+    
+    /** by default - do not cache  raw data in memory to preserve RAM */
+    private static final boolean cacheRaw = Boolean.valueOf( System.getProperty( SYSTEM_PROPERTY_CACHE_RAW,
"false" ) );
+
     private static final Language LANG = new DefaultLanguage( RepositoryGAVMetadata.class
);
 
     static volatile Map<String, MetadataCacheFs> fsCaches =
@@ -72,11 +82,12 @@
         (Map<String, RepositoryGAMetadata>) Collections.synchronizedMap( new HashMap<String,
RepositoryGAMetadata>( 512 ) );
 
     private volatile Map<String, RepositoryGAVMetadata> gavCache =
-        (Map<String, RepositoryGAVMetadata>) Collections.synchronizedMap( new HashMap<String,
RepositoryGAVMetadata>(
-                                                                                        
                             1024 ) );
+        (Map<String, RepositoryGAVMetadata>) Collections.synchronizedMap( new HashMap<String,
RepositoryGAVMetadata>( 1024 ) );
 
-    private volatile Map<String, byte[]> rawCache =
-        (Map<String, byte[]>) Collections.synchronizedMap( new HashMap<String, byte[]>(
1024 ) );
+    private volatile Map<String, byte[]> rawCache = cacheRaw
+        ? (Map<String, byte[]>) Collections.synchronizedMap( new HashMap<String,
byte[]>( 1024 ) )
+        : null
+        ;
 
     private File root;
 
@@ -374,7 +385,10 @@
             if ( _eventManager != null )
                 event = new GenericEvent( EventTypeEnum.fsCache, EVENT_FIND_RAW, rawKey );
 
-            byte[] res = rawCache.get( rawKey );
+            byte[] res = cacheRaw
+                ? rawCache.get( rawKey )
+                : null
+                ;
 
             if ( res != null )
             {
@@ -394,7 +408,8 @@
 
             res = FileUtil.readRawData( f );
 
-            rawCache.put( rawKey, res );
+            if( cacheRaw)
+                rawCache.put( rawKey, res );
 
             if ( _eventManager != null )
                 event.setResult( "found on disk" );
@@ -428,7 +443,8 @@
             if ( _eventManager != null )
                 event = new GenericEvent( EventTypeEnum.fsCache, EVENT_SAVE_RAW, rawKey );
 
-            rawCache.put( rawKey, rawBytes );
+            if( cacheRaw )
+                rawCache.put( rawKey, rawBytes );
 
             File f =
                 new File( getGAVDir( md.getEffectiveCoordinates() ), md.getArtifactId() +
FileUtil.DASH
@@ -519,7 +535,9 @@
     public void clearSession()
         throws MetadataCacheException
     {
-        rawCache.clear();
+        if( cacheRaw )
+            rawCache.clear();
+        
         gaCache.clear();
         gavCache.clear();
     }

Modified: maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java
URL: http://svn.apache.org/viewvc/maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java?rev=776133&r1=776132&r2=776133&view=diff
==============================================================================
--- maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java
(original)
+++ maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java
Mon May 18 23:17:24 2009
@@ -57,6 +57,8 @@
 import org.apache.maven.mercury.repository.cache.fs.MetadataCacheFs;
 import org.apache.maven.mercury.repository.remote.m2.RemoteRepositoryM2;
 import org.apache.maven.mercury.repository.remote.m2.RemoteRepositoryReaderM2;
+import org.apache.maven.mercury.util.LruMemCache;
+import org.apache.maven.mercury.util.MemCache;
 import org.apache.maven.mercury.util.Util;
 import org.codehaus.plexus.lang.DefaultLanguage;
 import org.codehaus.plexus.lang.Language;
@@ -119,6 +121,15 @@
     private boolean _initialized = false;
 
     private EventManager _eventManager;
+    
+    public static final String SYSTEM_PROPERTY_VERSION_CACHE_SIZE = "mercury.version.cache.size";
+    
+    private static final int _versionCacheSize = Integer.valueOf( System.getProperty( SYSTEM_PROPERTY_VERSION_CACHE_SIZE,
"1024" ) ); 
+    
+    private static final MemCache<ArtifactMetadata, List<ArtifactMetadata> >
_cachedVersions =
+        _versionCacheSize == 0 ? null
+        : new LruMemCache<ArtifactMetadata, List<ArtifactMetadata>>( _versionCacheSize
)
+        ;
 
     // ----------------------------------------------------------------------------------------------------------------------------
     public VirtualRepositoryReader( Collection<Repository> repositories )
@@ -301,6 +312,59 @@
         }
     }
 
+    //----------------------------------------------------------------------------------------------------------------------------
+    private MetadataResults readCachedVersions( Collection<ArtifactMetadata> query,
List<ArtifactMetadata> leftOvers )
+    {
+        if( _cachedVersions == null )
+        {
+            leftOvers.addAll( query );
+            return null;
+        }
+        
+        MetadataResults res = null;
+        
+        for( ArtifactMetadata key : query )
+        {
+            List<ArtifactMetadata> vl = _cachedVersions.get( key );
+            if( Util.isEmpty( vl ) )
+                leftOvers.add( key );
+            else
+            {
+                if( res == null )
+                    res = new MetadataResults( key, vl );
+                else
+                    res.add( key, vl );
+            }
+        }
+        
+        return res;
+    }
+    //----------------------------------------------------------------------------------------------------------------------------
+    private MetadataResults cacheVersions( MetadataResults res )
+    {
+        if(true)
+            return res;
+        
+        if( _cachedVersions == null || res == null || res.hasExceptions() || ! res.hasResults()
)
+        {
+            return res;
+        }
+        
+        for( ArtifactMetadata key :res.getResults().keySet() )
+        {
+            List<ArtifactMetadata> vl = res.getResult( key );
+            
+            if( Util.isEmpty( vl ) )
+                continue;
+            
+            for( ArtifactMetadata md : vl )
+                md.setTracker( null );
+            
+            _cachedVersions.put( key, vl );
+        }
+        
+        return res;
+    }
     // ----------------------------------------------------------------------------------------------------------------------------
     public MetadataResults readVersions( Collection<ArtifactMetadata> query )
         throws IllegalArgumentException, RepositoryException
@@ -321,13 +385,16 @@
                 event = new GenericEvent( EventTypeEnum.virtualRepositoryReader, EVENT_READ_VERSIONS
);
             }
 
-            MetadataResults res = null;
             ArtifactListProcessor tp = _processors == null ? null : _processors.get( ArtifactListProcessor.FUNCTION_TP
);
 
             GenericEvent eventRead = null;
 
             List<ArtifactMetadata> qList = new ArrayList<ArtifactMetadata>( query.size()
);
-            qList.addAll( query );
+//            qList.addAll( query );
+            MetadataResults res = readCachedVersions( query, qList );
+            
+            if( Util.isEmpty( qList ) )
+                return res;
 
             for ( RepositoryReader rr : _repositoryReaders )
             {
@@ -425,7 +492,7 @@
                 processSingletons( res );
             }
 
-            return res;
+            return cacheVersions( res );
         }
         finally
         {

Added: maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
URL: http://svn.apache.org/viewvc/maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java?rev=776133&view=auto
==============================================================================
--- maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
(added)
+++ maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
Mon May 18 23:17:24 2009
@@ -0,0 +1,71 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+
+package org.apache.maven.mercury.util;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ *
+ *
+ * @author Oleg Gusakov
+ * @version $Id$
+ *
+ */
+public class LruMemCache<K, V>
+    implements MemCache<K, V>
+{
+    ConcurrentHashMap<K, V> _cache;
+    
+    K _lastKey;
+    
+    int _sz;
+    
+    public LruMemCache()
+    {
+        this( DEFAULT_CACHE_SIZE );
+    }
+    
+    public LruMemCache( int sz )
+    {
+        _sz = sz;
+        _cache = new ConcurrentHashMap<K, V>( sz );
+    }
+
+    public V get( K key )
+    {
+        return _cache.get( key );
+    }
+
+    public void put( K key, V val )
+    {
+        // MRU is easier - do it first ..
+        synchronized( _cache )
+        {
+            if( _cache.size() == _sz )
+            {// free up one slot
+                _cache.remove( _lastKey );
+                _lastKey = key;
+            }
+        }
+
+        _cache.put( key, val );
+    }
+
+}

Propchange: maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
------------------------------------------------------------------------------
    svn:keywords = Author Date Id Revision

Added: maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
URL: http://svn.apache.org/viewvc/maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java?rev=776133&view=auto
==============================================================================
--- maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
(added)
+++ maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
Mon May 18 23:17:24 2009
@@ -0,0 +1,36 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+
+package org.apache.maven.mercury.util;
+
+/**
+ *
+ *
+ * @author Oleg Gusakov
+ * @version $Id$
+ *
+ */
+public interface MemCache<K extends Object, V extends Object>
+{
+    public static final int DEFAULT_CACHE_SIZE = 1024;
+    
+    void put( K key, V val );
+    
+    V get( K key );
+}

Propchange: maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
------------------------------------------------------------------------------
    svn:keywords = Author Date Id Revision

Modified: maven/mercury/trunk/mercury-it/pom.xml
URL: http://svn.apache.org/viewvc/maven/mercury/trunk/mercury-it/pom.xml?rev=776133&r1=776132&r2=776133&view=diff
==============================================================================
--- maven/mercury/trunk/mercury-it/pom.xml (original)
+++ maven/mercury/trunk/mercury-it/pom.xml Mon May 18 23:17:24 2009
@@ -182,7 +182,7 @@
   </dependencies>
 
   <build>
- 
+ <!-- 
     <filters>
       <filter>src/test/filters/filter.txt</filter>
     </filters>
@@ -209,7 +209,8 @@
 
     </testResources>
 
- 
+  -->
+  
     <plugins>
 
       <plugin>

Propchange: maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom
            ('svn:eol-style' removed)

Propchange: maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Modified: maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1
URL: http://svn.apache.org/viewvc/maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1?rev=776133&r1=776132&r2=776133&view=diff
==============================================================================
--- maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1
(original)
+++ maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1
Mon May 18 23:17:24 2009
@@ -1 +1 @@
-526bfebc865ac047ff3fa7d77924a4edff7ba468
\ No newline at end of file
+5ab8cde950b8b29bc5c8d79b8df5a804a76e573e



Mime
View raw message