Mailing-List: contact commits-help@activemq.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@activemq.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r677715 - in
 /activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer:
 BTreeFilerBenchmark.java FilerBenchmark.java HashFilerBenchmark.java
Date: Thu, 17 Jul 2008 20:41:42 -0000
To: commits@activemq.apache.org
From: chirino@apache.org
Message-Id: <20080717204142.99CA3238896E@eris.apache.org>

Author: chirino
Date: Thu Jul 17 13:41:41 2008
New Revision: 677715

URL: http://svn.apache.org/viewvc?rev=677715&view=rev
Log:
Adding a little benchmark tool that can be used to measure the performance of the Filer implementations.

Added:
    activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/BTreeFilerBenchmark.java
    activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/FilerBenchmark.java   (with props)
    activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/HashFilerBenchmark.java

Added: activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/BTreeFilerBenchmark.java
URL: http://svn.apache.org/viewvc/activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/BTreeFilerBenchmark.java?rev=677715&view=auto
==============================================================================
--- activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/BTreeFilerBenchmark.java (added)
+++ activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/BTreeFilerBenchmark.java Thu Jul 17 13:41:41 2008
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.xindice.core.filer;
+
+import java.io.File;
+
+public class BTreeFilerBenchmark extends FilerBenchmark {
+
+    @Override
+    protected Filer createFiler(File root, String name) throws Exception {
+        BTreeFiler rc = new BTreeFiler();
+        rc.setLocation(ROOT_DIR, name);
+        rc.setPageCount(10000);
+        rc.setPageSize(4096);
+        return rc;
+    }
+
+}

Added: activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/FilerBenchmark.java
URL: http://svn.apache.org/viewvc/activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/FilerBenchmark.java?rev=677715&view=auto
==============================================================================
--- activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/FilerBenchmark.java (added)
+++ activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/FilerBenchmark.java Thu Jul 17 13:41:41 2008
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.xindice.core.filer;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import junit.framework.TestCase;
+import org.apache.xindice.core.data.Key;
+import org.apache.xindice.core.data.Record;
+import org.apache.xindice.core.data.Value;
+
+/**
+ * @author chirino
+ */
+public abstract class FilerBenchmark extends TestCase {
+
+    // Slower machines might need to make this bigger.
+    private static final long SAMPLE_DURATION = Integer.parseInt(System.getProperty("SAMPLES_DURATION", "" + 1000 * 5));
+    // How many times do we sample?
+    private static final long SAMPLES = Integer.parseInt(System.getProperty("SAMPLES", "" + 60 * 1000 / SAMPLE_DURATION)); 
+
+    private static final int FILER_COUNT = Integer.parseInt(System.getProperty("FILER_COUNT", "" + 1));
+    private static final int FILER_PRE_LOAD_COUNT = Integer.parseInt(System.getProperty("FILER_PRE_LOAD_COUNT", "" + 10000 / FILER_COUNT));
+
+    protected File ROOT_DIR;
+    protected final HashMap<String, Filer> indexes = new HashMap<String, Filer>();
+
+    public void setUp() throws Exception {
+        ROOT_DIR = new File("target/test-data/" + getClass().getName());
+        ROOT_DIR.mkdirs();
+    }
+
+    protected void tearDown() throws Exception {
+        for (Filer f : indexes.values()) {
+            try {
+                f.close();
+            } catch (Throwable ignore) {
+            }
+        }
+    }
+
+    abstract protected Filer createFiler(File root, String name) throws Exception;
+
+    synchronized private Filer openFiler(String name) throws Exception, FilerException {
+        Filer index = indexes.get(name);
+        if (index == null) {
+            index = createFiler(ROOT_DIR, name);
+            if (!index.exists()) {
+                index.create();
+            }
+            index.open();
+            indexes.put(name, index);
+        }
+        return index;
+    }
+
+    class Producer extends Thread {
+        private final String name;
+        AtomicBoolean shutdown = new AtomicBoolean();
+
+        public Producer(String name) {
+            super("Producer: " + name);
+            this.name = name;
+        }
+
+        public void shutdown() {
+            shutdown.set(true);
+        }
+
+        @Override
+        public void run() {
+            try {
+                Filer filer = openFiler(name);
+                long counter = 0;
+                Value value = new Value(new byte[] {
+                    1, 2, 3, 4, 5, 6, 7, 8
+                });
+                while (!shutdown.get()) {
+                    long c = counter;
+                    Key key = new Key("a-long-message-id-like-key-" + c);
+                    filer.writeRecord(key, value);
+                    onProduced(counter++);
+                }
+
+            } catch (Throwable e) {
+                e.printStackTrace();
+            }
+        }
+
+        public void onProduced(long counter) {
+        }
+    }
+
+    class Consumer extends Thread {
+        private final String name;
+        AtomicBoolean shutdown = new AtomicBoolean();
+
+        public Consumer(String name) {
+            super("Consumer: " + name);
+            this.name = name;
+        }
+
+        public void shutdown() {
+            shutdown.set(true);
+        }
+
+        @Override
+        public void run() {
+            try {
+                Filer filer = openFiler(name);
+                long counter = 0;
+                while (!shutdown.get()) {
+                    long c = counter;
+                    Key key = new Key("a-long-message-id-like-key-" + c);
+                    Record record;
+                    record = filer.readRecord(key);
+                    if (record != null) {
+                        filer.deleteRecord(key);
+                        onConsumed(counter++);
+                    } else {
+                        Thread.sleep(0);
+                    }
+                }
+            } catch (Throwable e) {
+                e.printStackTrace();
+            }
+        }
+
+        public void onConsumed(long counter) {
+        }
+    }
+
+    public void testLoad() throws Exception {
+
+        final Producer producers[] = new Producer[FILER_COUNT];
+        final Consumer consumers[] = new Consumer[FILER_COUNT];
+        final CountDownLatch preloadCountDown = new CountDownLatch(FILER_COUNT);
+        final AtomicLong producedRecords = new AtomicLong();
+        final AtomicLong consumedRecords = new AtomicLong();
+
+        System.out.println("Starting: " + FILER_COUNT + " producers");
+        for (int i = 0; i < FILER_COUNT; i++) {
+            producers[i] = new Producer("test-" + i) {
+                private boolean prelaodDone;
+
+                public void onProduced(long counter) {
+                    if (!prelaodDone && counter >= FILER_PRE_LOAD_COUNT) {
+                        prelaodDone = true;
+                        preloadCountDown.countDown();
+                    }
+                    producedRecords.incrementAndGet();
+                }
+            };
+            producers[i].start();
+        }
+
+        long start = System.currentTimeMillis();
+        System.out.println("Waiting for each producer create " + FILER_PRE_LOAD_COUNT + " records before starting the consumers.");
+        preloadCountDown.await();
+        long end = System.currentTimeMillis();
+        System.out.println("Preloaded " + FILER_PRE_LOAD_COUNT * FILER_COUNT + " records at " + (FILER_PRE_LOAD_COUNT * FILER_COUNT * 1000f / (end - start)) + " records/sec");
+
+        System.out.println("Starting: " + FILER_COUNT + " consumers");
+        for (int i = 0; i < FILER_COUNT; i++) {
+            consumers[i] = new Consumer("test-" + i) {
+                public void onConsumed(long counter) {
+                    consumedRecords.incrementAndGet();
+                }
+            };
+            consumers[i].start();
+        }
+
+        long sample_start = System.currentTimeMillis();
+        System.out.println("Taking " + SAMPLES + " performance samples every " + SAMPLE_DURATION + " ms");
+        System.out.println("time (s), produced, produce rate (r/s), consumed, consume rate (r/s), used memory (k)");
+        producedRecords.set(0);
+        consumedRecords.set(0);
+        for (int i = 0; i < SAMPLES; i++) {
+            start = System.currentTimeMillis();
+            Thread.sleep(SAMPLE_DURATION);
+            end = System.currentTimeMillis();
+            long p = producedRecords.getAndSet(0);
+            long c = consumedRecords.getAndSet(0);
+
+            long usedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
+
+            System.out.println(((end - sample_start) / 1000f) + ", " + p + ", " + (p * 1000f / (end - start)) + ", " + c + ", " + (c * 1000f / (end - start)) + ", " + (usedMemory / (1024)));
+        }
+        System.out.println("Samples done... Shutting down the producers and consumers...");
+        for (int i = 0; i < FILER_COUNT; i++) {
+            producers[i].shutdown();
+            consumers[i].shutdown();
+        }
+        for (int i = 0; i < FILER_COUNT; i++) {
+            producers[i].join(1000 * 5);
+            consumers[i].join(1000 * 5);
+        }
+        System.out.println("Shutdown.");
+    }
+
+}

Propchange: activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/FilerBenchmark.java
------------------------------------------------------------------------------
    svn:executable = *

Added: activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/HashFilerBenchmark.java
URL: http://svn.apache.org/viewvc/activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/HashFilerBenchmark.java?rev=677715&view=auto
==============================================================================
--- activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/HashFilerBenchmark.java (added)
+++ activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/HashFilerBenchmark.java Thu Jul 17 13:41:41 2008
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.xindice.core.filer;
+
+import java.io.File;
+
+public class HashFilerBenchmark extends FilerBenchmark {
+
+    @Override
+    protected Filer createFiler(File root, String name) throws Exception {
+        HashFiler rc = new HashFiler();
+        rc.setLocation(ROOT_DIR, name);
+        rc.setPageCount(10000);
+        rc.setPageSize(4096);
+        return rc;
+    }
+
+}