incubator-kato-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From monte...@apache.org
Subject svn commit: r781076 - in /incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src: main/java/org/apache/kato/hprof/datalayer/ test/java/test/apache/kato/common/
Date Tue, 02 Jun 2009 16:28:38 GMT
Author: monteith
Date: Tue Jun  2 16:28:38 2009
New Revision: 781076

URL: http://svn.apache.org/viewvc?rev=781076&view=rev
Log:
Add caching to the file accessing. Approx 9x quicker...

Added:
    incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/main/java/org/apache/kato/hprof/datalayer/CachedRandomAccesDataProvider.java
  (with props)
    incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/test/java/test/apache/kato/common/TestCachedRandomAccessDataProvider.java
  (with props)
Modified:
    incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/main/java/org/apache/kato/hprof/datalayer/HProfFactory.java

Added: incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/main/java/org/apache/kato/hprof/datalayer/CachedRandomAccesDataProvider.java
URL: http://svn.apache.org/viewvc/incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/main/java/org/apache/kato/hprof/datalayer/CachedRandomAccesDataProvider.java?rev=781076&view=auto
==============================================================================
--- incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/main/java/org/apache/kato/hprof/datalayer/CachedRandomAccesDataProvider.java
(added)
+++ incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/main/java/org/apache/kato/hprof/datalayer/CachedRandomAccesDataProvider.java
Tue Jun  2 16:28:38 2009
@@ -0,0 +1,384 @@
+/*******************************************************************************
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.kato.hprof.datalayer;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.lang.ref.SoftReference;
+import java.util.List;
+
+import javax.imageio.stream.FileImageInputStream;
+
+import org.apache.kato.common.IDataProvider;
+
+/**
+ * Caches accesses to a RandomAccessFile.
+ * Stores an array of SoftReferences to CacheLine objects.
+ *
+ * Cache addresses are arranged line so:
+ * 
+ *   |    tag    |  index  |  line |
+ *   63                            0 
+ *
+ */
+public class CachedRandomAccesDataProvider implements IDataProvider {
+	
+	@SuppressWarnings("unchecked")
+	public CachedRandomAccesDataProvider(File reader) throws IOException {
+		if(reader==null) throw new IllegalArgumentException("reader file is null");
+		if(reader.exists()==false) throw new IllegalArgumentException("reader file ["+reader.getAbsolutePath()+
" ] does not exist");
+		RandomAccessFile r=new RandomAccessFile(reader, "r");
+		raf=new FileImageInputStream(r);		
+		
+		lineBits = 10; // 2 ^ 10 == 1024
+		lineSize = 1 << lineBits;
+		lineMask = (1L<< lineBits)-1L;		
+		indexBits = 18; // 2^ 10 == 262144 entries		
+		indexMask = ((1L << indexBits)-1) << lineBits;
+		tagBits = Long.SIZE - lineBits - indexBits;
+		tagMask = ~(lineMask | indexMask);
+		
+		int numLines = 1 << indexBits;
+		
+		lines = new SoftReference[numLines];
+	}
+
+	@Override
+	public void close() throws IOException {
+		raf.close();
+		raf=null;
+		
+	}
+
+	/**
+	 * Represents a cache line.
+	 * consists of a byte array and a tag to identify it.
+	 *
+	 */
+	private static class CacheLine {
+		private byte line[];
+		private long tag;
+	
+		public CacheLine(long tag, byte[] line) {
+			this.line = line;
+			this.tag = tag;
+		}
+		
+		/**
+		 * Tag to identify cacheline. 
+		 * 
+		 * @return tag
+		 */
+		public long getTag() {
+			return tag;
+		}
+		
+		/**
+		 * Tag to identify cache line
+		 * 
+		 * @return byte[] 
+		 */
+		public byte[] getLine() {
+			return line;
+		}
+	}
+	
+	private CacheLine currentLine = null;
+	private int currentIndex = 0;
+	private SoftReference<CacheLine> lines[];
+	
+	private int lineBits;
+	private long lineMask;
+	private int lineSize;
+	private int indexBits;
+	private long indexMask;
+	private int indexShift;
+	private int tagBits;
+	private long tagMask;
+	
+	private long fileSize;
+	
+	private FileImageInputStream raf=null;
+	private long fileLength=0;
+	private long current=0;
+	private boolean moved=false;
+
+	private long accesses=0, misses=0;
+	
+	/**
+	 * Given an offset into the file, return the appropriate cache line.
+	 * 
+	 * @param location location in file (no alignment necessary)
+	 * @return byte[] of cache line. check size.
+	 * @throws IOException 
+	 */
+	public byte[] getLine(long location) throws IOException {
+		int index = (int)((location & indexMask) >> lineBits);
+		long tag = (location & tagMask);
+		accesses++;
+
+		/**
+		 * Fast path - odds are this will be used. 
+		 * 
+		 */
+		if (currentLine != null) {
+			// currentIndex is necessary, the common case is for the tags
+			// to be identical, but the indexes different.
+			if ((currentLine.getTag() == tag) && (currentIndex == index)) {
+				return currentLine.getLine();
+			}
+		}
+		
+		// Retrieve cache line
+		// tag is used to distinguish between collisions
+		SoftReference<CacheLine> reference = lines[index];
+		
+		// references are created when needed.
+		if (reference == null) {
+			misses++;
+			CacheLine line = createCacheLine(location); 
+			reference = new SoftReference<CacheLine>(line);
+			lines[index] = reference;
+			currentIndex = index;
+			currentLine = line;
+		} else {
+			CacheLine line = reference.get();
+			
+			// Handles cases when cache line has wrong tag or
+			// when the cacheline has been garbage collected
+			if (line == null || (line.getTag() != tag)) {
+				misses++;
+				line = createCacheLine(location); 
+				reference = new SoftReference<CacheLine>(line);
+				lines[index] = reference;
+				currentLine = line;
+				currentIndex = index;				
+			} else {				
+				// cache line is correct.
+				currentLine =  line;
+				currentIndex = index;
+			}
+		}
+		
+		return currentLine.getLine();
+	}
+	
+	/**
+	 * Return offset within cache line
+	 * 
+	 * @param location
+	 * @return offset in cacheline.
+	 */
+	private int getOffset(long location) {
+		return (int) (location & lineMask);
+	}
+
+	/**
+	 * Creates a cacheline for a given location, unconditionally.
+	 * Loads as much data as is present in the file. 
+	 * The last cacheline will probably be truncated.
+	 * 
+	 * @param location absolute position of cache line in file.
+	 * @return CacheLine
+	 * @throws IOException
+	 */
+	public CacheLine createCacheLine(long location) throws IOException {
+		byte[] bytes;
+		location = location & (indexMask | tagMask);
+		
+		// possibly truncate the last cache line.
+		if (location + lineSize > fileLength) {
+			bytes = new byte[(int)(fileLength-location)];
+		} else {
+			bytes = new byte[lineSize];
+		}
+		raf.seek(location);				
+		raf.readFully(bytes);
+		
+		return new CacheLine(location & tagMask, bytes);
+	}	
+	
+	@Override
+	public void moveTo(long l) throws IOException {		
+		current=l;
+		moved=true;		
+	}
+
+	@Override
+	public void open() throws IOException {
+		fileLength=raf.length();
+		raf.seek(0);
+		current = 0;
+	}
+
+	@Override
+	public String readCString() throws IOException {		
+		StringBuilder builder=new StringBuilder();
+		// FIXME This could be quicker.
+		while(true) {
+			int b=readByte();
+			if(b==0) break;
+			builder.append( (char)b);
+		}
+		
+		return builder.toString();
+	}
+
+	@Override
+	public int readU4() throws IOException {
+		byte[] line = getLine(current);
+		int offset = getOffset(current);
+		
+		// Fast, common case.
+		if (offset+4 < line.length) {
+			current +=4;
+			return (int) ((line[offset] << 24) | 
+					((line[offset+1] & 0xff) << 16) |
+					((line[offset+2] & 0xff) << 8) |
+					((line[offset+3] & 0xff))
+					);
+		}
+		
+		int val = 0;
+		for (int i=0; i<4; i++) {
+			val = val << 8;
+			val = val | readByte();
+		}
+		// readByte() updates current for us.
+		
+		return val;
+	}
+
+	@Override
+	public long readU8() throws IOException {
+		byte[] line = getLine(current);
+		int offset = getOffset(current);
+		
+		// Fast, common case.
+		if (offset+8 < line.length) {
+			current +=8;
+			return (long)(
+					((line[offset  ] & 0xff) << 56) |
+					((line[offset+1] & 0xff) << 48) |
+					((line[offset+2] & 0xff) << 40) |
+					((line[offset+3] & 0xff) << 32) |
+					((line[offset+4] & 0xff) << 24) | 
+					((line[offset+5] & 0xff) << 16) |
+					((line[offset+6] & 0xff) << 8) |
+					((line[offset+7] & 0xff))
+					);
+		}
+		
+		int val = 0;
+		for (int i=0; i<8; i++) {
+			val = val << 8;
+			val = val | readByte();
+		}
+		
+		return val;
+	}
+
+	@Override
+	public long getCurrentLocation() throws IOException {		
+		return current;
+	}
+
+	@Override
+	public long getDataLength() throws IOException {
+		return fileLength;
+	}
+
+	@Override
+	public short readByte() throws IOException {
+		byte[] line = getLine(current);
+		int offset = getOffset(current);
+		
+		if (line == null) {
+			throw new IOException("Cannot resolve location "+current);
+		} else if (line.length < offset) {
+			throw new IOException("Location "+current+" has offset "+offset+" which is greater than
the line size "+line.length);
+		}
+		
+		short val = (short) (line[offset] & 0xff);
+		current++;
+		return val;
+	}
+
+	@Override
+	public byte[] readBytes(int left) throws IOException {
+//		updateLocation();
+		byte[] line = getLine(current);
+		int offset = getOffset(current);
+		byte[] data = new byte[left];
+		
+		if ((offset + left) < line.length ) {
+			current += left;
+			System.arraycopy(line, offset, data, 0, left);
+			return data;
+		} 
+		
+		// slow method - byte by byte
+		for(int i=0; i< left; i++) {
+			data[i] = (byte)readByte();			
+		}
+		
+		return data; 
+	}
+
+	@Override
+	public void moveBy(int move) throws IOException {
+		current+=move;
+		moved=true;	
+	}
+	
+	@Override
+	public short readU2() throws IOException {
+		byte[] line = getLine(current);
+		int offset = getOffset(current);
+		
+		// Fast, common case.
+		if (offset+2 < line.length) {
+			current +=2;
+			return (short) ((line[offset] << 8) | (line[offset+1] &0xff));
+		}
+		
+		short val = (short) ((readByte() << 8) | readByte());
+		return val;
+	}
+	
+	@Override
+	public void addState(List state) {
+		long realPos=-2;
+		try {
+			realPos = raf.getStreamPosition();
+		} catch (IOException e) {
+		
+		}
+		
+		state.add("RandomAccessDataProvider current="+current+",moved="+moved+",fileLength="+fileLength+",realpos="+realPos);
+		
+	}
+
+	@Override
+	public long dataRemaining() {
+		
+		return fileLength-current;
+	}
+	
+	@Override
+	public boolean moved() {
+		return moved;
+	}
+}

Propchange: incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/main/java/org/apache/kato/hprof/datalayer/CachedRandomAccesDataProvider.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/main/java/org/apache/kato/hprof/datalayer/HProfFactory.java
URL: http://svn.apache.org/viewvc/incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/main/java/org/apache/kato/hprof/datalayer/HProfFactory.java?rev=781076&r1=781075&r2=781076&view=diff
==============================================================================
--- incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/main/java/org/apache/kato/hprof/datalayer/HProfFactory.java
(original)
+++ incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/main/java/org/apache/kato/hprof/datalayer/HProfFactory.java
Tue Jun  2 16:28:38 2009
@@ -23,9 +23,8 @@
 	
 	
 	public static HProfFile createReader(File reader) throws IOException {
-		
-		IDataProvider provider=new RandomAccesDataProvider(reader);
-		
+		IDataProvider provider =new CachedRandomAccesDataProvider(reader);
+	
 		return new HProfFile(provider);
 		
 	}

Added: incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/test/java/test/apache/kato/common/TestCachedRandomAccessDataProvider.java
URL: http://svn.apache.org/viewvc/incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/test/java/test/apache/kato/common/TestCachedRandomAccessDataProvider.java?rev=781076&view=auto
==============================================================================
--- incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/test/java/test/apache/kato/common/TestCachedRandomAccessDataProvider.java
(added)
+++ incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/test/java/test/apache/kato/common/TestCachedRandomAccessDataProvider.java
Tue Jun  2 16:28:38 2009
@@ -0,0 +1,61 @@
+/*******************************************************************************
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package test.apache.kato.common;
+
+import java.io.File;
+import java.io.RandomAccessFile;
+
+import org.apache.kato.hprof.datalayer.CachedRandomAccesDataProvider;
+
+import junit.framework.TestCase;
+
+public class TestCachedRandomAccessDataProvider extends TestCase {
+	static File testFile = new File(System.getProperty("java.io.tmpdir")+File.separatorChar+"testFile");
+	
+	
+	public void setUp() throws Exception {
+		if(!testFile.exists()) {
+			RandomAccessFile raf = new RandomAccessFile(testFile,"rw");
+			
+			for(int i=0; i < 1048576; i++) {
+				raf.writeInt(i);
+			}
+			
+			raf.close();
+		}
+	}
+	
+	public void testRead() throws Exception {
+		CachedRandomAccesDataProvider provider = new CachedRandomAccesDataProvider(testFile);
+		provider.open();
+		
+		for(int i=0; i < 1048576; i++) {
+			int val = provider.readU4();
+			assertEquals(i,val);
+		}
+		provider.close();
+	}
+	
+	public void testReadBackwards() throws Exception {
+		CachedRandomAccesDataProvider provider = new CachedRandomAccesDataProvider(testFile);
+		provider.open();
+		
+		for(int i=1048572; i >0 ; i-=1) {
+			provider.moveTo(i*4);
+			int val = provider.readU4();
+			assertEquals(i,val);
+		}
+		provider.close();
+	}
+}

Propchange: incubator/kato/branches/experimental/maven_restructure/org.apache.kato/kato.hprof.reader/src/test/java/test/apache/kato/common/TestCachedRandomAccessDataProvider.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain



Mime
View raw message