Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 129D210E9C for ; Thu, 12 Dec 2013 00:27:08 +0000 (UTC) Received: (qmail 85503 invoked by uid 500); 12 Dec 2013 00:27:07 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 85324 invoked by uid 500); 12 Dec 2013 00:27:07 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 85294 invoked by uid 99); 12 Dec 2013 00:27:06 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Dec 2013 00:27:06 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 60D5A8159F6; Thu, 12 Dec 2013 00:27:06 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ctubbsii@apache.org To: commits@accumulo.apache.org Date: Thu, 12 Dec 2013 00:27:06 -0000 Message-Id: <5a55000b26804b9db460ce7690e8461c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/4] git commit: ACCUMULO-1981 Convert inert class to unit test Updated Branches: refs/heads/1.6.0-SNAPSHOT 5f90d0b8a -> c1fbeac50 ACCUMULO-1981 Convert inert class to unit test Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/96956216 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/96956216 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/96956216 Branch: refs/heads/1.6.0-SNAPSHOT Commit: 96956216c406eb7f2ba4f149933afcb5a02c9edd Parents: 7b7521d Author: Christopher Tubbs Authored: Mon Dec 9 13:38:07 2013 -0500 Committer: Christopher Tubbs Committed: Wed Dec 11 16:59:48 2013 -0500 ---------------------------------------------------------------------- .../core/file/BloomFilterLayerLookupTest.java | 121 ++++++++++--------- 1 file changed, 63 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/96956216/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java b/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java index 3078daf..24bde38 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java @@ -16,8 +16,10 @@ */ package org.apache.accumulo.core.file; +import static org.junit.Assert.assertTrue; + +import java.io.File; import java.io.IOException; -import java.io.PrintStream; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -27,114 +29,117 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.file.keyfunctor.ColumnFamilyFunctor; import org.apache.accumulo.core.file.rfile.RFile; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.io.Text; +import org.apache.log4j.Logger; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestName; public class BloomFilterLayerLookupTest { - public static void main(String[] args) throws IOException { - PrintStream out = System.out; - - Random r = new Random(); - + + private static final Logger LOG = Logger.getLogger(BloomFilterLayerLookupTest.class); + private static Random random = new Random(); + + @Rule + public TestName testName = new TestName(); + + @Rule + public TemporaryFolder tempDir = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target")); + + @Test + public void test() throws IOException { HashSet valsSet = new HashSet(); - for (int i = 0; i < 100000; i++) { - valsSet.add(r.nextInt(Integer.MAX_VALUE)); + valsSet.add(random.nextInt(Integer.MAX_VALUE)); } - + ArrayList vals = new ArrayList(valsSet); Collections.sort(vals); - + ConfigurationCopy acuconf = new ConfigurationCopy(AccumuloConfiguration.getDefaultConfiguration()); acuconf.set(Property.TABLE_BLOOM_ENABLED, "true"); - acuconf.set(Property.TABLE_BLOOM_KEY_FUNCTOR, "accumulo.core.file.keyfunctor.ColumnFamilyFunctor"); + acuconf.set(Property.TABLE_BLOOM_KEY_FUNCTOR, ColumnFamilyFunctor.class.getName()); acuconf.set(Property.TABLE_FILE_TYPE, RFile.EXTENSION); acuconf.set(Property.TABLE_BLOOM_LOAD_THRESHOLD, "1"); acuconf.set(Property.TSERV_BLOOM_LOAD_MAXCONCURRENT, "1"); - + Configuration conf = CachedConfiguration.getInstance(); FileSystem fs = FileSystem.get(conf); - + + // get output file name String suffix = FileOperations.getNewFileExtension(acuconf); - String fname = "/tmp/test." + suffix; + String fname = new File(tempDir.getRoot(), testName + "." + suffix).getAbsolutePath(); FileSKVWriter bmfw = FileOperations.getInstance().openWriter(fname, fs, conf, acuconf); - + + // write data to file long t1 = System.currentTimeMillis(); - bmfw.startDefaultLocalityGroup(); - for (Integer i : vals) { String fi = String.format("%010d", i); - bmfw.append(new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf1")), new Value(("v" + fi).getBytes())); - bmfw.append(new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf2")), new Value(("v" + fi).getBytes())); + bmfw.append(new Key(new Text("r" + fi), new Text("cf1")), new Value(("v" + fi).getBytes())); + bmfw.append(new Key(new Text("r" + fi), new Text("cf2")), new Value(("v" + fi).getBytes())); } - long t2 = System.currentTimeMillis(); - - out.printf("write rate %6.2f%n", vals.size() / ((t2 - t1) / 1000.0)); - + + LOG.debug(String.format("write rate %6.2f%n", vals.size() / ((t2 - t1) / 1000.0))); bmfw.close(); - + t1 = System.currentTimeMillis(); FileSKVIterator bmfr = FileOperations.getInstance().openReader(fname, false, fs, conf, acuconf); t2 = System.currentTimeMillis(); - out.println("Opened " + fname + " in " + (t2 - t1)); - - t1 = System.currentTimeMillis(); - + LOG.debug("Opened " + fname + " in " + (t2 - t1)); + int hits = 0; + t1 = System.currentTimeMillis(); for (int i = 0; i < 5000; i++) { - int row = r.nextInt(Integer.MAX_VALUE); - String fi = String.format("%010d", row); - // bmfr.seek(new Range(new Text("r"+fi))); - org.apache.accumulo.core.data.Key k1 = new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf1")); - bmfr.seek(new Range(k1, true, k1.followingKey(PartialKey.ROW_COLFAM), false), new ArrayList(), false); + int row = random.nextInt(Integer.MAX_VALUE); + seek(bmfr, row); if (valsSet.contains(row)) { hits++; - if (!bmfr.hasTop()) { - out.println("ERROR " + row); - } + assertTrue(bmfr.hasTop()); } } - t2 = System.currentTimeMillis(); - - out.printf("random lookup rate : %6.2f%n", 5000 / ((t2 - t1) / 1000.0)); - out.println("hits = " + hits); - + + double rate1 = 5000 / ((t2 - t1) / 1000.0); + LOG.debug(String.format("random lookup rate : %6.2f%n", rate1)); + LOG.debug("hits = " + hits); + int count = 0; - t1 = System.currentTimeMillis(); - for (Integer row : valsSet) { - String fi = String.format("%010d", row); - // bmfr.seek(new Range(new Text("r"+fi))); - - org.apache.accumulo.core.data.Key k1 = new org.apache.accumulo.core.data.Key(new Text("r" + fi), new Text("cf1")); - bmfr.seek(new Range(k1, true, k1.followingKey(PartialKey.ROW_COLFAM), false), new ArrayList(), false); - - if (!bmfr.hasTop()) { - out.println("ERROR 2 " + row); - } - + seek(bmfr, row); + assertTrue(bmfr.hasTop()); count++; - if (count >= 500) { break; } } - t2 = System.currentTimeMillis(); - - out.printf("existant lookup rate %6.2f%n", 500 / ((t2 - t1) / 1000.0)); - out.println("expected hits 500. Receive hits: " + count); + + double rate2 = 500 / ((t2 - t1) / 1000.0); + LOG.debug(String.format("existant lookup rate %6.2f%n", rate2)); + LOG.debug("expected hits 500. Receive hits: " + count); bmfr.close(); + + assertTrue(rate1 > rate2); + } + + private void seek(FileSKVIterator bmfr, int row) throws IOException { + String fi = String.format("%010d", row); + // bmfr.seek(new Range(new Text("r"+fi))); + Key k1 = new Key(new Text("r" + fi), new Text("cf1")); + bmfr.seek(new Range(k1, true, k1.followingKey(PartialKey.ROW_COLFAM), false), new ArrayList(), false); } }