commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t.@apache.org
Subject svn commit: r1716376 [1/3] - in /commons/proper/jcs/trunk: commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/ commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/indexed/ commons-jcs-core/src/main/java/org/apac...
Date Wed, 25 Nov 2015 13:10:23 GMT
Author: tv
Date: Wed Nov 25 13:10:22 2015
New Revision: 1716376

URL: http://svn.apache.org/viewvc?rev=1716376&view=rev
Log:
JCS-153: Fix file size limitation for Block Disk Cache and Indexed Disk Cache

Added:
    commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheCountUnitTest.java
  (with props)
    commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheSizeUnitTest.java
  (with props)
    commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTestAbstract.java
  (with props)
    commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexDiskCacheCountUnitTest.java
  (with props)
    commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexDiskCacheSizeUnitTest.java
  (with props)
    commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTestAbstract.java
  (with props)
Removed:
    commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTest.java
    commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTestCount.java
    commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCacheUnitTestSize.java
    commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTest.java
    commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTestCount.java
    commons/proper/jcs/trunk/commons-jcs-core/src/test/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTestSize.java
Modified:
    commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java
    commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
    commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/indexed/IndexedDiskCache.java
    commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/utils/struct/AbstractLRUMap.java
    commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/utils/struct/LRUMap.java
    commons/proper/jcs/trunk/src/changes/changes.xml

Modified: commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java
URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java?rev=1716376&r1=1716375&r2=1716376&view=diff
==============================================================================
--- commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java
(original)
+++ commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskCache.java
Wed Nov 25 13:10:22 2015
@@ -125,11 +125,13 @@ public class BlockDiskCache<K, V>
             if ( this.blockDiskCacheAttributes.getBlockSizeBytes() > 0 )
             {
                 this.dataFile = new BlockDisk( new File( rootDirectory, fileName + ".data"
),
-                                               this.blockDiskCacheAttributes.getBlockSizeBytes()
);
+                                               this.blockDiskCacheAttributes.getBlockSizeBytes(),
+                                               getElementSerializer() );
             }
             else
             {
-                this.dataFile = new BlockDisk( new File( rootDirectory, fileName + ".data"
), getElementSerializer() );
+                this.dataFile = new BlockDisk( new File( rootDirectory, fileName + ".data"
),
+                                               getElementSerializer() );
             }
 
             keyStore = new BlockDiskKeyStore<K>( this.blockDiskCacheAttributes, this
);

Modified: commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskKeyStore.java?rev=1716376&r1=1716375&r2=1716376&view=diff
==============================================================================
--- commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
(original)
+++ commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs/auxiliary/disk/block/BlockDiskKeyStore.java
Wed Nov 25 13:10:22 2015
@@ -44,12 +44,13 @@ import org.apache.commons.logging.LogFac
 /**
  * This is responsible for storing the keys.
  * <p>
+ *
  * @author Aaron Smuts
  */
 public class BlockDiskKeyStore<K>
 {
     /** The logger */
-    private static final Log log = LogFactory.getLog( BlockDiskKeyStore.class );
+    private static final Log log = LogFactory.getLog(BlockDiskKeyStore.class);
 
     /** Attributes governing the behavior of the block disk cache. */
     private final BlockDiskCacheAttributes blockDiskCacheAttributes;
@@ -79,35 +80,36 @@ public class BlockDiskKeyStore<K>
     /**
      * Set the configuration options.
      * <p>
+     *
      * @param cacheAttributes
-     * @param blockDiskCache used for freeing
+     * @param blockDiskCache
+     *            used for freeing
      */
-    public BlockDiskKeyStore( BlockDiskCacheAttributes cacheAttributes,
-            BlockDiskCache<K, ?> blockDiskCache)
+    public BlockDiskKeyStore(BlockDiskCacheAttributes cacheAttributes, BlockDiskCache<K,
?> blockDiskCache)
     {
         this.blockDiskCacheAttributes = cacheAttributes;
         this.logCacheName = "Region [" + this.blockDiskCacheAttributes.getCacheName() + "]
";
         this.fileName = this.blockDiskCacheAttributes.getCacheName();
         this.maxKeySize = cacheAttributes.getMaxKeySize();
         this.blockDiskCache = blockDiskCache;
-        this.diskLimitType  = cacheAttributes.getDiskLimitType();
+        this.diskLimitType = cacheAttributes.getDiskLimitType();
         this.blockSize = cacheAttributes.getBlockSizeBytes();
 
         File rootDirectory = cacheAttributes.getDiskPath();
 
-        if ( log.isInfoEnabled() )
+        if (log.isInfoEnabled())
         {
-            log.info( logCacheName + "Cache file root directory [" + rootDirectory + "]"
);
+            log.info(logCacheName + "Cache file root directory [" + rootDirectory + "]");
         }
 
-        this.keyFile = new File( rootDirectory, fileName + ".key" );
+        this.keyFile = new File(rootDirectory, fileName + ".key");
 
-        if ( log.isInfoEnabled() )
+        if (log.isInfoEnabled())
         {
-            log.info( logCacheName + "Key File [" + this.keyFile.getAbsolutePath() + "]"
);
+            log.info(logCacheName + "Key File [" + this.keyFile.getAbsolutePath() + "]");
         }
 
-        if ( keyFile.length() > 0 )
+        if (keyFile.length() > 0)
         {
             loadKeys();
             // TODO verify somehow
@@ -128,27 +130,26 @@ public class BlockDiskKeyStore<K>
         {
             ElapsedTimer timer = new ElapsedTimer();
             int numKeys = keyHash.size();
-            if ( log.isInfoEnabled() )
+            if (log.isInfoEnabled())
             {
-                log.info( logCacheName + "Saving keys to [" + this.keyFile.getAbsolutePath()
+ "], key count ["
-                    + numKeys + "]" );
+                log.info(logCacheName + "Saving keys to [" + this.keyFile.getAbsolutePath()
+ "], key count [" + numKeys + "]");
             }
 
             synchronized (keyFile)
             {
-                FileOutputStream fos = new FileOutputStream( keyFile );
-                BufferedOutputStream bos = new BufferedOutputStream( fos, 65536 );
-                ObjectOutputStream oos = new ObjectOutputStream( bos );
+                FileOutputStream fos = new FileOutputStream(keyFile);
+                BufferedOutputStream bos = new BufferedOutputStream(fos, 65536);
+                ObjectOutputStream oos = new ObjectOutputStream(bos);
                 try
                 {
                     // don't need to synchronize, since the underlying collection makes a
copy
                     for (Map.Entry<K, int[]> entry : keyHash.entrySet())
                     {
                         BlockDiskElementDescriptor<K> descriptor = new BlockDiskElementDescriptor<K>();
-                        descriptor.setKey( entry.getKey() );
-                        descriptor.setBlocks( entry.getValue() );
+                        descriptor.setKey(entry.getKey());
+                        descriptor.setBlocks(entry.getValue());
                         // stream these out in the loop.
-                        oos.writeObject( descriptor );
+                        oos.writeUnshared(descriptor);
                     }
                 }
                 finally
@@ -158,15 +159,15 @@ public class BlockDiskKeyStore<K>
                 }
             }
 
-            if ( log.isInfoEnabled() )
+            if (log.isInfoEnabled())
             {
-                log.info( logCacheName + "Finished saving keys. It took " + timer.getElapsedTimeString()
+ " to store "
-                    + numKeys + " keys.  Key file length [" + keyFile.length() + "]" );
+                log.info(logCacheName + "Finished saving keys. It took " + timer.getElapsedTimeString()
+ " to store " + numKeys
+                    + " keys.  Key file length [" + keyFile.length() + "]");
             }
         }
-        catch ( IOException e )
+        catch (IOException e)
         {
-            log.error( logCacheName + "Problem storing keys.", e );
+            log.error(logCacheName + "Problem storing keys.", e);
         }
     }
 
@@ -180,7 +181,6 @@ public class BlockDiskKeyStore<K>
             clearMemoryMap();
             saveKeys();
         }
-
     }
 
     /**
@@ -197,16 +197,19 @@ public class BlockDiskKeyStore<K>
     private void initKeyMap()
     {
         keyHash = null;
-        if ( maxKeySize >= 0 )
+        if (maxKeySize >= 0)
         {
-            if (this.diskLimitType.equals(DiskLimitType.SIZE)) {
+            if (this.diskLimitType == DiskLimitType.SIZE)
+            {
                 keyHash = new LRUMapSizeLimited(maxKeySize);
-            } else {
-                keyHash = new LRUMapCountLimited( maxKeySize );
             }
-            if ( log.isInfoEnabled() )
+            else
             {
-                log.info( logCacheName + "Set maxKeySize to: '" + maxKeySize + "'" );
+                keyHash = new LRUMapCountLimited(maxKeySize);
+            }
+            if (log.isInfoEnabled())
+            {
+                log.info(logCacheName + "Set maxKeySize to: '" + maxKeySize + "'");
             }
         }
         else
@@ -214,9 +217,9 @@ public class BlockDiskKeyStore<K>
             // If no max size, use a plain map for memory and processing efficiency.
             keyHash = new HashMap<K, int[]>();
             // keyHash = Collections.synchronizedMap( new HashMap() );
-            if ( log.isInfoEnabled() )
+            if (log.isInfoEnabled())
             {
-                log.info( logCacheName + "Set maxKeySize to unlimited'" );
+                log.info(logCacheName + "Set maxKeySize to unlimited'");
             }
         }
     }
@@ -227,9 +230,9 @@ public class BlockDiskKeyStore<K>
      */
     protected void loadKeys()
     {
-        if ( log.isInfoEnabled() )
+        if (log.isInfoEnabled())
         {
-            log.info( logCacheName + "Loading keys for " + keyFile.toString() );
+            log.info(logCacheName + "Loading keys for " + keyFile.toString());
         }
 
         try
@@ -241,22 +244,23 @@ public class BlockDiskKeyStore<K>
 
             synchronized (keyFile)
             {
-                FileInputStream fis = new FileInputStream( keyFile );
-                BufferedInputStream bis = new BufferedInputStream( fis );
-                ObjectInputStream ois = new ObjectInputStreamClassLoaderAware( bis , null);
+                FileInputStream fis = new FileInputStream(keyFile);
+                BufferedInputStream bis = new BufferedInputStream(fis, 65536);
+                ObjectInputStream ois = new ObjectInputStreamClassLoaderAware(bis, null);
                 try
                 {
-                    while ( true )
+                    while (true)
                     {
-                        @SuppressWarnings("unchecked") // Need to cast from Object
+                        @SuppressWarnings("unchecked")
+                        // Need to cast from Object
                         BlockDiskElementDescriptor<K> descriptor = (BlockDiskElementDescriptor<K>)
ois.readObject();
-                        if ( descriptor != null )
+                        if (descriptor != null)
                         {
-                            keys.put( descriptor.getKey(), descriptor.getBlocks() );
+                            keys.put(descriptor.getKey(), descriptor.getBlocks());
                         }
                     }
                 }
-                catch ( EOFException eof )
+                catch (EOFException eof)
                 {
                     // nothing
                 }
@@ -266,31 +270,32 @@ public class BlockDiskKeyStore<K>
                 }
             }
 
-            if ( !keys.isEmpty() )
+            if (!keys.isEmpty())
             {
-                keyHash.putAll( keys );
+                keyHash.putAll(keys);
 
-                if ( log.isDebugEnabled() )
+                if (log.isDebugEnabled())
                 {
-                    log.debug( logCacheName + "Found " + keys.size() + " in keys file." );
+                    log.debug(logCacheName + "Found " + keys.size() + " in keys file.");
                 }
 
-                if ( log.isInfoEnabled() )
+                if (log.isInfoEnabled())
                 {
-                    log.info( logCacheName + "Loaded keys from [" + fileName + "], key count:
" + keyHash.size()
-                        + "; up to " + maxKeySize + " will be available." );
+                    log.info(logCacheName + "Loaded keys from [" + fileName + "], key count:
" + keyHash.size() + "; up to "
+                        + maxKeySize + " will be available.");
                 }
             }
         }
-        catch ( Exception e )
+        catch (Exception e)
         {
-            log.error( logCacheName + "Problem loading keys for file " + fileName, e );
+            log.error(logCacheName + "Problem loading keys for file " + fileName, e);
         }
     }
 
     /**
      * Gets the entry set.
      * <p>
+     *
      * @return entry set.
      */
     public Set<Map.Entry<K, int[]>> entrySet()
@@ -301,6 +306,7 @@ public class BlockDiskKeyStore<K>
     /**
      * Gets the key set.
      * <p>
+     *
      * @return key set.
      */
     public Set<K> keySet()
@@ -311,6 +317,7 @@ public class BlockDiskKeyStore<K>
     /**
      * Gets the size of the key hash.
      * <p>
+     *
      * @return the number of keys.
      */
     public int size()
@@ -321,91 +328,123 @@ public class BlockDiskKeyStore<K>
     /**
      * gets the object for the key.
      * <p>
+     *
      * @param key
      * @return Object
      */
-    public int[] get( K key )
+    public int[] get(K key)
     {
-        return this.keyHash.get( key );
+        return this.keyHash.get(key);
     }
 
     /**
      * Puts a int[] in the keyStore.
      * <p>
+     *
      * @param key
      * @param value
      */
-    public void put( K key, int[] value )
+    public void put(K key, int[] value)
     {
-        this.keyHash.put( key, value );
+        this.keyHash.put(key, value);
     }
 
     /**
      * Remove by key.
      * <p>
+     *
      * @param key
      * @return BlockDiskElementDescriptor if it was present, else null
      */
-    public int[] remove( K key )
+    public int[] remove(K key)
     {
-        return this.keyHash.remove( key );
+        return this.keyHash.remove(key);
     }
 
-
     /**
      * Class for recycling and lru. This implements the LRU size overflow callback, so we
can mark the
      * blocks as free.
      */
-
-    public class LRUMapSizeLimited
-    	extends AbstractLRUMap<K, int[]>
-
+    public class LRUMapSizeLimited extends AbstractLRUMap<K, int[]>
     {
         /**
          * <code>tag</code> tells us which map we are working on.
          */
-        public String tag = "orig-lru-size";
+        public final String tag = "orig-lru-size";
+
         // size of the content in kB
-        private AtomicInteger contentSize = new AtomicInteger();
-        private int maxSize = -1;
+        private AtomicInteger contentSize;
+        private int maxSize;
+
         /**
          * Default
          */
         public LRUMapSizeLimited()
         {
-            super();
+            this(-1);
         }
 
         /**
-         * @param maxKeySize
+         * @param maxSize maximum cache size in kB
          */
-        public LRUMapSizeLimited( int maxKeySize )
+        public LRUMapSizeLimited(int maxSize)
         {
             super();
-            this.maxSize = maxKeySize;
+            this.maxSize = maxSize;
+            this.contentSize = new AtomicInteger(0);
+        }
+
+        // keep the content size in kB, so 2^31 kB is reasonable value
+        private void subLengthFromCacheSize(int[] value)
+        {
+            contentSize.addAndGet(value.length * blockSize / -1024 - 1);
+        }
+
+        // keep the content size in kB, so 2^31 kB is reasonable value
+        private void addLengthToCacheSize(int[] value)
+        {
+            contentSize.addAndGet(value.length * blockSize / 1024 + 1);
         }
 
         @Override
-        public int[] put(K key, int[] value) {
-            try {
-                return super.put(key, value);
-            } finally {
-                // keep the content size in kB, so 2^31 kB is reasonable value
-                contentSize.addAndGet((int) Math.ceil(value.length * blockSize / 1024.0));
+        public int[] put(K key, int[] value)
+        {
+            int[] oldValue = null;
+
+            try
+            {
+                oldValue = super.put(key, value);
             }
+            finally
+            {
+                if (value != null)
+                {
+                    addLengthToCacheSize(value);
+                }
+                if (oldValue != null)
+                {
+                    subLengthFromCacheSize(oldValue);
+                }
+            }
+
+            return oldValue;
         }
 
         @Override
-        public int[] remove(Object key ) {
+        public int[] remove(Object key)
+        {
             int[] value = null;
 
-            try {
+            try
+            {
                 value = super.remove(key);
                 return value;
-            } finally {
-                if (value != null) {
-                    // keep the content size in kB, so 2^31 kB is reasonable value
-                    contentSize.addAndGet((int) ((Math.ceil(value.length * blockSize / 1024.0))
* -1));
+            }
+            finally
+            {
+                if (value != null)
+                {
+                    subLengthFromCacheSize(value);
                 }
             }
         }
@@ -414,57 +453,65 @@ public class BlockDiskKeyStore<K>
          * This is called when the may key size is reached. The least recently used item
will be
          * passed here. We will store the position and size of the spot on disk in the recycle
bin.
          * <p>
+         *
          * @param key
          * @param value
          */
         @Override
-        protected void processRemovedLRU( K key, int[] value )
+        protected void processRemovedLRU(K key, int[] value)
         {
-            blockDiskCache.freeBlocks( value );
-            if ( log.isDebugEnabled() )
+            blockDiskCache.freeBlocks(value);
+            if (log.isDebugEnabled())
             {
-                log.debug( logCacheName + "Removing key: [" + key + "] from key store." );
-                log.debug( logCacheName + "Key store size: [" + super.size() + "]." );
+                log.debug(logCacheName + "Removing key: [" + key + "] from key store.");
+                log.debug(logCacheName + "Key store size: [" + super.size() + "].");
+            }
+
+            if (value != null)
+            {
+                subLengthFromCacheSize(value);
             }
         }
+
         @Override
-        protected boolean shouldRemove() {
-            return maxSize > 0 && contentSize.intValue() > maxSize &&
this.size() > 1;
+        protected boolean shouldRemove()
+        {
+            return maxSize > 0 && contentSize.get() > maxSize && this.size()
> 1;
         }
     }
+
     /**
      * Class for recycling and lru. This implements the LRU overflow callback, so we can
mark the
      * blocks as free.
      */
-    public class LRUMapCountLimited
-    extends LRUMap<K, int[]>
-    // implements Serializable
+    public class LRUMapCountLimited extends LRUMap<K, int[]>
     {
         /**
          * <code>tag</code> tells us which map we are working on.
          */
-        public String tag = "orig-lru-count";
+        public final String tag = "orig-lru-count";
 
-        public LRUMapCountLimited(int maxKeySize) {
+        public LRUMapCountLimited(int maxKeySize)
+        {
             super(maxKeySize);
         }
 
-
         /**
          * This is called when the may key size is reached. The least recently used item
will be
          * passed here. We will store the position and size of the spot on disk in the recycle
bin.
          * <p>
+         *
          * @param key
          * @param value
          */
         @Override
-        protected void processRemovedLRU( K key, int[] value )
+        protected void processRemovedLRU(K key, int[] value)
         {
-            blockDiskCache.freeBlocks( value );
-            if ( log.isDebugEnabled() )
+            blockDiskCache.freeBlocks(value);
+            if (log.isDebugEnabled())
             {
-                log.debug( logCacheName + "Removing key: [" + key + "] from key store." );
-                log.debug( logCacheName + "Key store size: [" + super.size() + "]." );
+                log.debug(logCacheName + "Removing key: [" + key + "] from key store.");
+                log.debug(logCacheName + "Key store size: [" + super.size() + "].");
             }
         }
     }



Mime
View raw message