Return-Path: X-Original-To: apmail-incubator-directmemory-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-directmemory-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 324FF91E3 for ; Thu, 13 Oct 2011 10:17:35 +0000 (UTC) Received: (qmail 92080 invoked by uid 500); 13 Oct 2011 10:17:34 -0000 Delivered-To: apmail-incubator-directmemory-commits-archive@incubator.apache.org Received: (qmail 92058 invoked by uid 500); 13 Oct 2011 10:17:34 -0000 Mailing-List: contact directmemory-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: directmemory-dev@incubator.apache.org Delivered-To: mailing list directmemory-commits@incubator.apache.org Received: (qmail 92043 invoked by uid 99); 13 Oct 2011 10:17:33 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 13 Oct 2011 10:17:33 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 13 Oct 2011 10:17:24 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 9C155238888F; Thu, 13 Oct 2011 10:17:02 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1182758 - in /incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache: ./ directmemory/ directmemory/cache/ directmemory/measures/ directmemory/memory/ directmemory/misc/ directmemory/serialization/ Date: Thu, 13 Oct 2011 10:17:01 -0000 To: directmemory-commits@incubator.apache.org From: olamy@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20111013101702.9C155238888F@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: olamy Date: Thu Oct 13 10:17:00 2011 New Revision: 1182758 URL: http://svn.apache.org/viewvc?rev=1182758&view=rev Log: arff missed again svn add: time to lunch break :-) Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/cache/ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/cache/Cache.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Every.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Expires.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/For.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Heap.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/In.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Memory.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Monitor.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Ram.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Sizing.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Space.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/MemoryManager.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/OffHeapMemoryBuffer.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/Pointer.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/DummyPojo.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/Format.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/DummyPojoSerializer.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffSerializerV1.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffWithLinkedBufferSerializer.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/Serializer.java (with props) incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/StandardSerializer.java (with props) Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/cache/Cache.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/cache/Cache.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/cache/Cache.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/cache/Cache.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,225 @@ +package org.apache.directmemory.cache; + +import java.io.EOFException; +import java.io.IOException; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentMap; + +import org.apache.directmemory.measures.Every; +import org.apache.directmemory.measures.Ram; +import org.apache.directmemory.memory.MemoryManager; +import org.apache.directmemory.memory.OffHeapMemoryBuffer; +import org.apache.directmemory.memory.Pointer; +import org.apache.directmemory.misc.Format; +import org.apache.directmemory.serialization.ProtoStuffSerializerV1; +import org.apache.directmemory.serialization.Serializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.MapMaker; + +public class Cache { + + private static Logger logger = LoggerFactory.getLogger(MemoryManager.class); + private static ConcurrentMap map; + + public static int DEFAULT_CONCURRENCY_LEVEL = 4; + public static int DEFAULT_INITIAL_CAPACITY = 100000; + + public static Serializer serializer = new ProtoStuffSerializerV1(); + + private Cache() { + // not instantiable + } + + + private final static Timer timer = new Timer(); + + public static void scheduleDisposalEvery(long l) { + timer.schedule(new TimerTask() { + public void run() { + logger.info("begin scheduled disposal"); + collectExpired(); + collectLFU(); + logger.info("scheduled disposal complete"); + } + }, l); + logger.info("disposal scheduled every " + l + " milliseconds"); + } + + public static void init(int numberOfBuffers, int size, int initialCapacity, int concurrencyLevel) { + map = new MapMaker() + .concurrencyLevel(concurrencyLevel) + .initialCapacity(initialCapacity) + .makeMap(); + + logger.info("*** initializing *******************************\r\n" + Format.logo()); + logger.info("************************************************"); + MemoryManager.init(numberOfBuffers, size); + logger.info("initialized"); + logger.info(Format.it("number of buffer(s): \t%1d with %2s each", numberOfBuffers, Ram.inMb(size))); + logger.info(Format.it("initial capacity: \t%1d", initialCapacity)); + logger.info(Format.it("concurrency level: \t%1d", concurrencyLevel)); + scheduleDisposalEvery(Every.seconds(10)); + } + + public static void init(int numberOfBuffers, int size) { + init(numberOfBuffers, size, DEFAULT_INITIAL_CAPACITY, DEFAULT_CONCURRENCY_LEVEL); + } + + public static Pointer putByteArray(String key, byte[] payload, int expiresIn) { + Pointer ptr = MemoryManager.store(payload, expiresIn); + map.put(key, ptr); + return ptr; + } + + public static Pointer putByteArray(String key, byte[] payload) { + return putByteArray(key, payload, 0); + } + + public static Pointer put(String key, Object object) { + return put(key, object, 0); + } + + public static Pointer put(String key, Object object, int expiresIn) { + try { + byte[] payload = serializer.serialize(object, object.getClass()); + Pointer ptr = putByteArray(key, payload); + ptr.clazz = object.getClass(); + return ptr; + } catch (IOException e) { + logger.error(e.getMessage()); + return null; + } + } + + public static Pointer updateByteArray(String key, byte[] payload) { + Pointer p = map.get(key); + p = MemoryManager.update(p, payload); + return p; + } + + public static Pointer update(String key, Object object) { + Pointer p = map.get(key); + try { + p = MemoryManager.update(p, serializer.serialize(object, object.getClass())); + p.clazz = object.getClass(); + return p; + } catch (IOException e) { + logger.error(e.getMessage()); + return null; + } + } + + public static byte[] retrieveByteArray(String key) { + Pointer ptr = getPointer(key); + if (ptr == null) return null; + if (ptr.expired() || ptr.free) { + map.remove(key); + if (!ptr.free) { + MemoryManager.free(ptr); + } + return null; + } else { + return MemoryManager.retrieve(ptr); + } + } + + public static Object retrieve(String key) { + Pointer ptr = getPointer(key); + if (ptr == null) return null; + if (ptr.expired() || ptr.free) { + map.remove(key); + if (!ptr.free) { + MemoryManager.free(ptr); + } + return null; + } else { + try { + return serializer.deserialize(MemoryManager.retrieve(ptr),ptr.clazz); + } catch (EOFException e) { + logger.error(e.getMessage()); + } catch (IOException e) { + logger.error(e.getMessage()); + } catch (ClassNotFoundException e) { + logger.error(e.getMessage()); + } catch (InstantiationException e) { + logger.error(e.getMessage()); + } catch (IllegalAccessException e) { + logger.error(e.getMessage()); + } + } + return null; + } + + public static Pointer getPointer(String key) { + return map.get(key); + } + + public static void free(String key) { + Pointer p = map.remove(key); + if (p != null) { + MemoryManager.free(p); + } + } + + public static void free(Pointer pointer) { + MemoryManager.free(pointer); + } + + public static void collectExpired() { + MemoryManager.collectExpired(); + // still have to look for orphan (storing references to freed pointers) map entries + } + + public static void collectLFU() { + MemoryManager.collectLFU(); + // can possibly clear one whole buffer if it's too fragmented - investigate + } + + public static void collectAll() { + Thread thread = new Thread(){ + public void run(){ + logger.info("begin disposal"); + collectExpired(); + collectLFU(); + logger.info("disposal complete"); + } + }; + thread.start(); + } + + + public static void clear() { + map.clear(); + MemoryManager.clear(); + logger.info("Cache cleared"); + } + + public static long entries() { + return map.size(); + } + + private static void dump(OffHeapMemoryBuffer mem) { + logger.info(Format.it("off-heap - buffer: \t%1d", mem.bufferNumber)); + logger.info(Format.it("off-heap - allocated: \t%1s", Ram.inMb(mem.capacity()))); + logger.info(Format.it("off-heap - used: \t%1s", Ram.inMb(mem.used()))); + logger.info(Format.it("heap - max: \t%1s", Ram.inMb(Runtime.getRuntime().maxMemory()))); + logger.info(Format.it("heap - allocated: \t%1s", Ram.inMb(Runtime.getRuntime().totalMemory()))); + logger.info(Format.it("heap - free : \t%1s", Ram.inMb(Runtime.getRuntime().freeMemory()))); + logger.info("************************************************"); + } + + public static void dump() { + if (!logger.isInfoEnabled()) + return; + + logger.info("*** DirectMemory statistics ********************"); + + for (OffHeapMemoryBuffer mem : MemoryManager.buffers) { + dump(mem); + } + } + +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/cache/Cache.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/cache/Cache.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Every.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Every.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Every.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Every.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,9 @@ +package org.apache.directmemory.measures; + +public class Every extends In { + + public Every(double measure) { + super(measure); + } + +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Every.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Every.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Expires.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Expires.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Expires.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Expires.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,15 @@ +package org.apache.directmemory.measures; + +public class Expires extends In { + + public Expires(double measure) { + super(measure); + } + + public static In in(double measure) { + return new In(measure); + } + public static long never() { + return -1L; + } +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Expires.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Expires.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/For.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/For.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/For.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/For.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,9 @@ +package org.apache.directmemory.measures; + +public class For extends In { + + public For(double measure) { + super(measure); + } + +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/For.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/For.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Heap.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Heap.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Heap.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Heap.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,5 @@ +package org.apache.directmemory.measures; + +public class Heap extends Sizing { + +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Heap.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Heap.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/In.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/In.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/In.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/In.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,48 @@ +package org.apache.directmemory.measures; + +public class In { + private double measure; + + public In(double measure) { + this.measure = measure; + } + + public long seconds() { + return seconds(measure); + } + + public long minutes() { + return minutes(measure); + } + + public long hours() { + return hours(measure); + } + + public long days() { + return days(measure); + } + + public static long seconds(double seconds) { + return (long)seconds * 1000; + } + public static long minutes(double minutes) { + return (long)seconds(minutes * 60); + } + public static long hours(double hours) { + return (long)minutes(hours * 60); + } + public static long days(double days) { + return (long)hours(days * 24); + } + + public static In just(double measure) { + return new In(measure); + } + public static In exactly(double measure) { + return new In(measure); + } + public static In only(double measure) { + return new In(measure); + } +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/In.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/In.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Memory.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Memory.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Memory.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Memory.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,5 @@ +package org.apache.directmemory.measures; + +public class Memory extends Sizing { + +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Memory.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Memory.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Monitor.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Monitor.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Monitor.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Monitor.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,75 @@ +package org.apache.directmemory.measures; + +import java.text.DecimalFormat; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.directmemory.misc.Format; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Monitor { + private AtomicLong hits = new AtomicLong(0); + private long totalTime = 0; + private long min = -1; + private long max = -1; + public String name; + + private static Logger logger = LoggerFactory.getLogger(Monitor.class); + public static Map monitors = new HashMap(); + + public static Monitor get(String key) { + Monitor mon = monitors.get(key); + if (mon == null) { + mon = new Monitor(key); + monitors.put(key, mon); + } + return mon; + } + + public Monitor(String name) { + this.name = name; + } + + public long start() { + return System.nanoTime(); + } + public long stop(long begunAt) { + hits.incrementAndGet(); + final long lastAccessed = System.nanoTime(); + final long elapsed = lastAccessed - begunAt; + totalTime+=elapsed; + if (elapsed > max && hits.get() > 0) max = elapsed; + if (elapsed < min && hits.get() > 0) min = elapsed; + return elapsed; + } + public long hits() { + return hits.get(); + } + public long totalTime() { + return totalTime; + } + public long average() { + return totalTime/hits.get(); + } + public String toString() { + return Format.it("%1$s hits: %2$d, avg: %3$s ms, tot: %4$s seconds", + name, + hits.get(), + new DecimalFormat("####.###").format((double)average()/1000000), + new DecimalFormat("####.###").format((double)totalTime/1000000000) + ); + } + + public static void dump(String prefix) { + for (Monitor monitor : Monitor.monitors.values()) { + if (monitor.name.startsWith(prefix)) + logger.info(monitor.toString()); + } + } + + public static void dump() { + dump(""); + } +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Monitor.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Monitor.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Ram.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Ram.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Ram.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Ram.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,5 @@ +package org.apache.directmemory.measures; + +public class Ram extends Sizing { + +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Ram.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Ram.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Sizing.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Sizing.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Sizing.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Sizing.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,29 @@ +package org.apache.directmemory.measures; + +import java.util.Formatter; + +public class Sizing { + public static int Gb(double giga) { + return (int)giga * 1024 * 1024 * 1024; + } + + public static int Mb(double mega) { + return (int)mega * 1024 * 1024; + } + + public static int Kb(double kilo) { + return (int)kilo * 1024; + } + public static int unlimited() { + return -1; + } + public static String inKb(long bytes) { + return new Formatter().format("%(,.1fKb", (double)bytes/1024).toString(); + } + public static String inMb(long bytes) { + return new Formatter().format("%(,.1fMb", (double)bytes/1024/1024).toString(); + } + public static String inGb(long bytes) { + return new Formatter().format("%(,.1fKb", (double)bytes/1024/1024/1024).toString(); + } +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Sizing.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Sizing.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Space.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Space.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Space.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Space.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,5 @@ +package org.apache.directmemory.measures; + +public class Space extends Sizing { + +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Space.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/measures/Space.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/MemoryManager.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/MemoryManager.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/MemoryManager.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/MemoryManager.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,97 @@ +package org.apache.directmemory.memory; + +import java.util.List; +import java.util.Vector; + +import org.apache.directmemory.measures.Ram; +import org.apache.directmemory.misc.Format; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MemoryManager { + private static Logger logger = LoggerFactory.getLogger(MemoryManager.class); + public static List buffers = new Vector(); + public static OffHeapMemoryBuffer activeBuffer = null; + + private MemoryManager() { + //static class + } + + public static void init(int numberOfBuffers, int size) { + for (int i = 0; i < numberOfBuffers; i++) { + buffers.add(OffHeapMemoryBuffer.createNew(size, i)); + } + activeBuffer = buffers.get(0); + logger.info(Format.it("MemoryManager initialized - %d buffers, %s each", numberOfBuffers, Ram.inMb(size))); + } + + public static Pointer store(byte[] payload, int expiresIn) { + Pointer p = activeBuffer.store(payload, expiresIn); + if (p == null) { + if (activeBuffer.bufferNumber+1 == buffers.size()) { + return null; + } else { + // try next buffer + activeBuffer = buffers.get(activeBuffer.bufferNumber+1); + p = activeBuffer.store(payload, expiresIn); + } + } + return p; + } + + public static Pointer store(byte[] payload) { + return store(payload, 0); + } + + public static Pointer update(Pointer pointer, byte[] payload) { + Pointer p = activeBuffer.update(pointer, payload); + if (p == null) { + if (activeBuffer.bufferNumber == buffers.size()) { + return null; + } else { + // try next buffer + activeBuffer = buffers.get(activeBuffer.bufferNumber+1); + p = activeBuffer.store(payload); + } + } + return p; + } + + public static byte[] retrieve(Pointer pointer) { + return buffers.get(pointer.bufferNumber).retrieve(pointer); + } + + public static void free(Pointer pointer) { + buffers.get(pointer.bufferNumber).free(pointer); + } + + public static void clear() { + for (OffHeapMemoryBuffer buffer : buffers) { + buffer.clear(); + } + activeBuffer = buffers.get(0); + } + + public static long capacity() { + long totalCapacity = 0; + for (OffHeapMemoryBuffer buffer : buffers) { + totalCapacity += buffer.capacity(); + } + return totalCapacity; + } + + public static long collectExpired() { + long disposed = 0; + for (OffHeapMemoryBuffer buffer : buffers) { + disposed += buffer.collectExpired(); + } + return disposed; + } + + public static void collectLFU() { + for (OffHeapMemoryBuffer buf : MemoryManager.buffers) { + buf.collectLFU(-1); + } + } + +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/MemoryManager.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/MemoryManager.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/OffHeapMemoryBuffer.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/OffHeapMemoryBuffer.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/OffHeapMemoryBuffer.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/OffHeapMemoryBuffer.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,251 @@ +package org.apache.directmemory.memory; + +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.CRC32; +import java.util.zip.Checksum; + +import org.apache.directmemory.measures.Ram; +import org.apache.directmemory.misc.Format; +import org.josql.Query; +import org.josql.QueryExecutionException; +import org.josql.QueryParseException; +import org.josql.QueryResults; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OffHeapMemoryBuffer { + private static Logger logger = LoggerFactory.getLogger(OffHeapMemoryBuffer.class); + protected ByteBuffer buffer; + public List pointers = new ArrayList(); +// public List pointers = new CopyOnWriteArrayList(); + AtomicInteger used = new AtomicInteger(); + public int bufferNumber; + + + public int used() { + return used.get(); + } + + public int capacity(){ + return buffer.capacity(); + } + + public static OffHeapMemoryBuffer createNew(int capacity, int bufferNumber) { + logger.info(Format.it("Creating OffHeapMemoryBuffer %d with a capacity of %s", bufferNumber, Ram.inMb(capacity))); + return new OffHeapMemoryBuffer(ByteBuffer.allocateDirect(capacity), bufferNumber); + } + + public static OffHeapMemoryBuffer createNew(int capacity) { + return new OffHeapMemoryBuffer(ByteBuffer.allocateDirect(capacity), -1); + } + + private OffHeapMemoryBuffer(ByteBuffer buffer, int bufferNumber) { + this.buffer = buffer; + this.bufferNumber = bufferNumber; + createAndAddFirstPointer(); + } + + private Pointer createAndAddFirstPointer() { + Pointer first = new Pointer(); + first.bufferNumber = bufferNumber; + first.start = 0; + first.free = true; + first.end = buffer.capacity()-1; + pointers.add(first); + return first; + } + + public Pointer slice(Pointer existing, int capacity) { + Pointer fresh = new Pointer(); + fresh.bufferNumber = existing.bufferNumber; + fresh.start = existing.start; + fresh.end = fresh.start+capacity; + fresh.free = true; + existing.start+=capacity+1; + return fresh; + } + + + public Pointer firstMatch(int capacity) { + for (Pointer ptr : pointers) { + if (ptr.free && ptr.end >= capacity) { + return ptr; + } + } + return null; + } + + public Pointer store(byte[] payload) { + return store(payload, -1); + } + + public byte[] retrieve(Pointer pointer) { +// if (!pointer.expired()) { + pointer.lastHit = System.currentTimeMillis(); + pointer.hits++; + + ByteBuffer buf = null; + synchronized (buffer) { + buf = buffer.duplicate(); + } + buf.position(pointer.start); + // not needed for reads + // buf.limit(pointer.end+pointer.start); + final byte[] swp = new byte[pointer.end-pointer.start]; + buf.get(swp); + return swp; +// } else { +// free(pointer); +// return null; +// } + } + + + public long free(Pointer pointer2free) { + pointer2free.free = true; + pointer2free.created = 0; + pointer2free.lastHit = 0; + pointer2free.hits = 0; + pointer2free.expiresIn = 0; + pointer2free.clazz = null; + used.addAndGet(-( pointer2free.end-pointer2free.start)); + pointers.add(pointer2free); + return pointer2free.end-pointer2free.start; + } + + public void clear() { + pointers.clear(); + createAndAddFirstPointer(); + buffer.clear(); + used.set(0); + } + + public Pointer store(byte[] payload, Date expires) { + return store(payload, 0, expires.getTime()); + } + + public Pointer store(byte[] payload, long expiresIn) { + return store(payload, expiresIn, 0); + } + + private synchronized Pointer store(byte[] payload, long expiresIn, long expires) { + Pointer goodOne = firstMatch(payload.length); + + if (goodOne == null ) { + throw new NullPointerException("did not find a suitable buffer"); + } + + Pointer fresh = slice(goodOne, payload.length); + + + fresh.created = System.currentTimeMillis(); + if (expiresIn > 0) { + fresh.expiresIn = expiresIn; + fresh.expires = 0; + } else if (expires > 0) { + fresh.expiresIn = 0; + fresh.expires = expires; + } + + fresh.free = false; + used.addAndGet(payload.length); + ByteBuffer buf = buffer.slice(); + buf.position(fresh.start); + try { + buf.put(payload); + } catch (BufferOverflowException e) { + // RpG not convincing - let's fix it later + goodOne.start = fresh.start; + goodOne.end = buffer.limit(); + return null; + } + pointers.add(fresh); + return fresh; + } + + private QueryResults select(String whereClause) throws QueryParseException, QueryExecutionException { + Query q = new Query (); + q.parse ("SELECT * FROM " + Pointer.class.getCanonicalName() + " WHERE " + whereClause); + QueryResults qr = q.execute (pointers); + return qr; + } + + private QueryResults selectOrderBy(String whereClause, String orderBy, String limit) throws QueryParseException, QueryExecutionException { + Query q = new Query (); + q.parse ("SELECT * FROM " + Pointer.class.getCanonicalName() + " WHERE " + whereClause + " order by " + orderBy + " " + limit); + QueryResults qr = q.execute (pointers); + return qr; + } + + public long collectLFU(int limit) { + if (limit<=0) limit = pointers.size()/10; + QueryResults qr; + try { + qr = selectOrderBy("free=false", "frequency", "limit 1, " + limit); + @SuppressWarnings("unchecked") + List result = qr.getResults(); + return free(result); + } catch (QueryParseException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (QueryExecutionException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return 0; + } + + + + @SuppressWarnings("unchecked") + private List filter(final String whereClause) { + try { + return select(whereClause).getResults(); + } catch (QueryParseException e) { + e.printStackTrace(); + } catch (QueryExecutionException e) { + e.printStackTrace(); + } + return (List) new ArrayList(); + } + + private long free(List pointers) { + long howMuch = 0; + for (Pointer expired : pointers) { + howMuch += free(expired); + } + return howMuch; + } + + public void disposeExpiredRelative() { + free(filter("free=false and expiresIn > 0 and (expiresIn+created) <= " + System.currentTimeMillis())); + } + + public void disposeExpiredAbsolute() { + free(filter("free=false and expires > 0 and (expires) <= " + System.currentTimeMillis())); + } + + public long collectExpired() { + int limit = 50; + long disposed = free(filter("free=false and expiresIn > 0 and (expiresIn+created) <= " + System.currentTimeMillis() + " limit 1, " + limit)); + disposed += free(filter("free=false and expires > 0 and (expires) <= " + System.currentTimeMillis() + " limit 1, 100" + limit)); + return disposed; + } + + public static long crc32(byte[] payload) { + final Checksum checksum = new CRC32(); + checksum.update(payload,0,payload.length); + return checksum.getValue(); + } + + public Pointer update(Pointer pointer, byte[] payload) { + free(pointer); + return store(payload); + } + +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/OffHeapMemoryBuffer.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/OffHeapMemoryBuffer.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/Pointer.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/Pointer.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/Pointer.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/Pointer.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,29 @@ +package org.apache.directmemory.memory; + +public class Pointer { + public int start; + public int end; + public long created; + public long expires; + public long expiresIn; + public long hits; + public boolean free; + public long lastHit; + public int bufferNumber; + public Class clazz; + + public byte[] content() { + return null; + } + public boolean expired() { + if (expires > 0 || expiresIn > 0) { + return (expiresIn + created < System.currentTimeMillis()); + } else { + return false; + } + } + + public float getFrequency() { + return (float)(System.currentTimeMillis()-created)/hits; + } +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/Pointer.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/memory/Pointer.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/DummyPojo.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/DummyPojo.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/DummyPojo.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/DummyPojo.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,22 @@ +package org.apache.directmemory.misc; +import java.io.Serializable; + + +public class DummyPojo implements Serializable { + /** + * A dummy pojo implementation for test purposes + */ + private static final long serialVersionUID = 1L; + public DummyPojo() { + + } + + public DummyPojo(String name, int size) { + this.name = name; + this.size = size; + payLoad = new String(new byte [size]); + } + public String name; + public int size; + public String payLoad; +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/DummyPojo.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/DummyPojo.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/Format.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/Format.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/Format.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/Format.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,28 @@ +package org.apache.directmemory.misc; + +public class Format { + + public static String it(String string, Object ... args) { + java.util.Formatter formatter = new java.util.Formatter(); + return formatter.format(string, args).toString(); + } + + public static String logo() { + return +" ____ _ __ __ ___\r\n" + +" / __ \\(_)________ _____/ /_/ |/ /___ ____ ___ ____ _______ __\r\n" + +" / / / / // ___/ _ \\/ ___/ __/ /|_/ // _ \\/ __ `__ \\/ __ \\/ ___/ / / /\r\n" + +" / /_/ / // / / __/ /__/ /_/ / / // __/ / / / / / /_/ / / / /_/ / \r\n" + +" /_____/_//_/ \\___/\\___/\\__/_/ /_/ \\___/_/ /_/ /_/\\____/_/ \\__, /\r\n" + +" /____/ "; + +// return +// " ___ _ _ _\r\n" + +// " ( / \\ o _/_( / ) )\r\n" + +// " / /, _ _ _, / / / / _ _ _ _ __ _ __ ,\r\n" + +// "(/\\_/ (_/ (_(/_(__(__ / / (_(/_/ / / /_(_)/ (_/ (_/_\r\n" + +// " /\r\n" + +// " '"; + } + +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/Format.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/misc/Format.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/DummyPojoSerializer.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/DummyPojoSerializer.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/DummyPojoSerializer.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/DummyPojoSerializer.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,40 @@ +package org.apache.directmemory.serialization; + +import java.io.EOFException; +import java.io.IOException; + +import org.apache.directmemory.measures.Ram; +import org.apache.directmemory.misc.DummyPojo; + +import com.dyuproject.protostuff.LinkedBuffer; +import com.dyuproject.protostuff.ProtostuffIOUtil; +import com.dyuproject.protostuff.runtime.RuntimeSchema; + +public final class DummyPojoSerializer implements Serializer + { + + final DummyPojo pojo = new DummyPojo("test", Ram.Kb(2)); + final byte[] data; + + public DummyPojoSerializer() + { + data = ProtostuffIOUtil.toByteArray(pojo, RuntimeSchema.getSchema(DummyPojo.class), LinkedBuffer.allocate(2048)); + } + + @Override + public Object deserialize(byte[] source, @SuppressWarnings({"rawtypes","unchecked"}) Class clazz) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException, + EOFException + { + // testing puts only + return pojo; + } + + @Override + public byte[] serialize(Object obj, @SuppressWarnings({"rawtypes","unchecked"}) Class clazz) throws IOException + { +// byte[] ser = new byte[data.length]; +// System.arraycopy(data, 0, ser, 0, data.length); + return data; + } + + } \ No newline at end of file Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/DummyPojoSerializer.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/DummyPojoSerializer.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffSerializerV1.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffSerializerV1.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffSerializerV1.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffSerializerV1.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,46 @@ +package org.apache.directmemory.serialization; + +import java.io.IOException; + +import org.apache.directmemory.measures.Ram; + +import com.dyuproject.protostuff.LinkedBuffer; +import com.dyuproject.protostuff.ProtostuffIOUtil; +import com.dyuproject.protostuff.Schema; +import com.dyuproject.protostuff.runtime.RuntimeSchema; + +public class ProtoStuffSerializerV1 implements Serializer { + + static int serBufferSize = Ram.Kb(3); +// static int serBufferSize = 300; + + /* (non-Javadoc) + * @see org.apache.directmemory.utils.Serializer#serialize(java.lang.Object, java.lang.Class) + */ + @SuppressWarnings("unchecked") + public byte[] serialize(Object obj, @SuppressWarnings("rawtypes") Class clazz) throws IOException { + @SuppressWarnings("rawtypes") + Schema schema = RuntimeSchema.getSchema(clazz); + final LinkedBuffer buffer = LinkedBuffer.allocate(serBufferSize); + byte[] protostuff = null; + + try { + protostuff = ProtostuffIOUtil.toByteArray(obj, schema, buffer); + } finally { + buffer.clear(); + } + return protostuff; + } + + /* (non-Javadoc) + * @see org.apache.directmemory.utils.Serializer#deserialize(byte[], java.lang.Class) + */ + @SuppressWarnings("unchecked") + public Object deserialize(byte[] source, @SuppressWarnings("rawtypes") Class clazz) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { + final Object object = clazz.newInstance(); + @SuppressWarnings("rawtypes") + final Schema schema = RuntimeSchema.getSchema(clazz); + ProtostuffIOUtil.mergeFrom(source, object, schema); + return object; + } +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffSerializerV1.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffSerializerV1.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffWithLinkedBufferSerializer.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffWithLinkedBufferSerializer.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffWithLinkedBufferSerializer.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffWithLinkedBufferSerializer.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,81 @@ +package org.apache.directmemory.serialization; + +import java.io.IOException; + +import org.apache.directmemory.measures.Ram; + +import com.dyuproject.protostuff.LinkedBuffer; +import com.dyuproject.protostuff.ProtostuffIOUtil; +import com.dyuproject.protostuff.Schema; +import com.dyuproject.protostuff.runtime.RuntimeSchema; + +public final class ProtoStuffWithLinkedBufferSerializer implements Serializer { + + static int bufferSize = Ram.Kb(3); + + + /* + * + * + * + LinkedBuffer buffer8k = ...; + try + { + ProtostuffIOUtil.writeTo(new ByteBufferOutputStream() { // paging logic }, message, schema, buffer8k) + } + finally + { + buffer8k.clear(); + } + + ProtostuffIOUtil.mergeFrom(new ByteArrayInputStream() { // paging logic}, message, schema, buffer8k); + +*/ + + private static final ThreadLocal localBuffer = new ThreadLocal() { + protected LinkedBuffer initialValue() { + return LinkedBuffer.allocate(bufferSize); + } + }; + + public ProtoStuffWithLinkedBufferSerializer() { + + } + + + public ProtoStuffWithLinkedBufferSerializer(int bufferSize) { + ProtoStuffWithLinkedBufferSerializer.bufferSize =bufferSize; + } + + + + /* (non-Javadoc) + * @see org.apache.directmemory.utils.Serializer#serialize(java.lang.Object, java.lang.Class) + */ + @SuppressWarnings("unchecked") + public byte[] serialize(Object obj, @SuppressWarnings("rawtypes") Class clazz) throws IOException { + @SuppressWarnings("rawtypes") + Schema schema = RuntimeSchema.getSchema(clazz); + final LinkedBuffer buffer = localBuffer.get(); + byte[] protostuff = null; + + try { + protostuff = ProtostuffIOUtil.toByteArray(obj, schema, buffer); + } finally { + buffer.clear(); + } + return protostuff; + } + + /* (non-Javadoc) + * @see org.apache.directmemory.utils.Serializer#deserialize(byte[], java.lang.Class) + */ + @SuppressWarnings("unchecked") + public Object deserialize(byte[] source, @SuppressWarnings("rawtypes") Class clazz) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { + Object object = clazz.newInstance(); + @SuppressWarnings("rawtypes") + Schema schema = RuntimeSchema.getSchema(clazz); + ProtostuffIOUtil.mergeFrom(source, object, schema); + return object; + } +} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffWithLinkedBufferSerializer.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/ProtoStuffWithLinkedBufferSerializer.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/Serializer.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/Serializer.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/Serializer.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/Serializer.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,13 @@ +package org.apache.directmemory.serialization; + +import java.io.EOFException; +import java.io.IOException; + +public interface Serializer { + + public abstract byte[] serialize(Object obj, @SuppressWarnings({"rawtypes","unchecked"}) Class clazz) throws IOException; + + public abstract Object deserialize(byte[] source, @SuppressWarnings({"rawtypes","unchecked"}) Class clazz) throws IOException, + ClassNotFoundException, InstantiationException, IllegalAccessException, EOFException; + +} \ No newline at end of file Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/Serializer.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/Serializer.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision Added: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/StandardSerializer.java URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/StandardSerializer.java?rev=1182758&view=auto ============================================================================== --- incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/StandardSerializer.java (added) +++ incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/StandardSerializer.java Thu Oct 13 10:17:00 2011 @@ -0,0 +1,27 @@ +package org.apache.directmemory.serialization; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; + +public class StandardSerializer implements Serializer { + + public byte[] serialize(Object obj, @SuppressWarnings({"rawtypes","unchecked"}) Class clazz) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos); + oos.writeObject(obj); + oos.flush(); + oos.close(); + return baos.toByteArray(); + } + + public Serializable deserialize(byte[] source, @SuppressWarnings({"rawtypes","unchecked"}) Class clazz) throws IOException, ClassNotFoundException { + ByteArrayInputStream bis = new ByteArrayInputStream(source); + ObjectInputStream ois = new ObjectInputStream(bis); + Serializable obj = (Serializable) ois.readObject(); + ois.close(); + return obj; + }} Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/StandardSerializer.java ------------------------------------------------------------------------------ svn:eol-style = native Propchange: incubator/directmemory/trunk/directmemory-cache/src/main/java/org/apache/directmemory/serialization/StandardSerializer.java ------------------------------------------------------------------------------ svn:keywords = Author Date Id Revision