Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A189F1087A for ; Sat, 26 Oct 2013 00:36:32 +0000 (UTC) Received: (qmail 93527 invoked by uid 500); 26 Oct 2013 00:36:32 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 93502 invoked by uid 500); 26 Oct 2013 00:36:32 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 93417 invoked by uid 99); 26 Oct 2013 00:36:32 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 26 Oct 2013 00:36:32 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 1232581E9AA; Sat, 26 Oct 2013 00:36:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ctubbsii@apache.org To: commits@accumulo.apache.org Date: Sat, 26 Oct 2013 00:36:36 -0000 Message-Id: <2e523b9c4e884bbaa4f9aba7ee8c4362@git.apache.org> In-Reply-To: <6237b9865b7d46cf940c756b5e0396f2@git.apache.org> References: <6237b9865b7d46cf940c756b5e0396f2@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [6/6] git commit: ACCUMULO-1599 Use target directory more ACCUMULO-1599 Use target directory more Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e95dee9e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e95dee9e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e95dee9e Branch: refs/heads/master Commit: e95dee9eca6d87ebaee0c087b1ed0fad0a226a57 Parents: aa0dcf2 Author: Christopher Tubbs Authored: Fri Oct 25 18:20:50 2013 -0400 Committer: Christopher Tubbs Committed: Fri Oct 25 20:33:40 2013 -0400 ---------------------------------------------------------------------- core/pom.xml | 10 + .../crypto-on-accumulo-site.xml | 162 ++ ...rypto-on-no-key-encryption-accumulo-site.xml | 144 ++ .../accumulo/core/file/rfile/RFileTest.java | 1081 +++++++------ .../test/resources/crypto-on-accumulo-site.xml | 164 -- ...rypto-on-no-key-encryption-accumulo-site.xml | 144 -- .../minicluster/MiniAccumuloConfig.java | 11 +- .../apache/accumulo/proxy/SimpleProxyIT.java | 1525 ++++++++++++++++++ .../org/apache/accumulo/proxy/SimpleTest.java | 1513 ----------------- .../server/tabletserver/InMemoryMapTest.java | 245 +-- .../apache/accumulo/test/AccumuloDFSBase.java | 21 +- .../apache/accumulo/test/AuditMessageIT.java | 478 ++++++ .../apache/accumulo/test/AuditMessageTest.java | 503 ------ .../accumulo/test/ConditionalWriterIT.java | 1169 ++++++++++++++ .../accumulo/test/ConditionalWriterTest.java | 1237 -------------- .../org/apache/accumulo/test/MetaSplitIT.java | 76 + .../org/apache/accumulo/test/MetaSplitTest.java | 103 -- .../apache/accumulo/test/SplitRecoveryIT.java | 134 ++ .../test/TestAccumuloSplitRecovery.java | 154 -- .../java/org/apache/accumulo/test/VolumeIT.java | 115 ++ .../org/apache/accumulo/test/VolumeTest.java | 120 -- .../accumulo/test/functional/ClassLoaderIT.java | 18 +- 22 files changed, 4521 insertions(+), 4606 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/e95dee9e/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index 6d6ba9f..9ae5d15 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -139,6 +139,16 @@ + + + true + src/test/filtered-resources + + + false + src/test/resources + + http://git-wip-us.apache.org/repos/asf/accumulo/blob/e95dee9e/core/src/test/filtered-resources/crypto-on-accumulo-site.xml ---------------------------------------------------------------------- diff --git a/core/src/test/filtered-resources/crypto-on-accumulo-site.xml b/core/src/test/filtered-resources/crypto-on-accumulo-site.xml new file mode 100644 index 0000000..70345e0 --- /dev/null +++ b/core/src/test/filtered-resources/crypto-on-accumulo-site.xml @@ -0,0 +1,162 @@ + + + + + + + + + instance.zookeeper.host + localhost:2181 + comma separated list of zookeeper servers + + + + logger.dir.walog + walogs + The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories. + + + + instance.secret + DEFAULT + A secret unique to a given instance that all servers must know in order to communicate with one another. + Change it before initialization. To change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd], + and then update this file. + + + + + tserver.memory.maps.max + 80M + + + + tserver.cache.data.size + 7M + + + + tserver.cache.index.size + 20M + + + + trace.password + + password + + + + trace.user + root + + + + tserver.sort.buffer.size + 50M + + + + tserver.walog.max.size + 100M + + + + general.classpaths + + $ACCUMULO_HOME/server/target/classes/, + $ACCUMULO_HOME/core/target/classes/, + $ACCUMULO_HOME/start/target/classes/, + $ACCUMULO_HOME/fate/target/classes/, + $ACCUMULO_HOME/proxy/target/classes/, + $ACCUMULO_HOME/examples/target/classes/, + $ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar, + $ACCUMULO_HOME/lib/[^.].*.jar, + $ZOOKEEPER_HOME/zookeeper[^.].*.jar, + $HADOOP_CONF_DIR, + $HADOOP_PREFIX/[^.].*.jar, + $HADOOP_PREFIX/lib/[^.].*.jar, + + Classpaths that accumulo checks for updates and class files. + When using the Security Manager, please remove the ".../target/classes/" values. + + + + + crypto.module.class + org.apache.accumulo.core.security.crypto.DefaultCryptoModule + + + crypto.cipher.suite + AES/CFB/PKCS5Padding + + + crypto.cipher.algorithm.name + AES + + + crypto.cipher.key.length + 128 + + + crypto.secure.rng + SHA1PRNG + + + crypto.secure.rng.provider + SUN + + + crypto.secret.key.encryption.strategy.class + org.apache.accumulo.core.security.crypto.DefaultSecretKeyEncryptionStrategy + + + instance.dfs.dir + ${project.build.directory} + + + instance.dfs.uri + file://${project.build.directory} + + + + crypto.default.key.strategy.hdfs.uri + file://${project.build.directory} + + + crypto.default.key.strategy.key.location + cryptoTestFile/test.secret.key + + + + crypto.default.key.strategy.cipher.suite + AES/ECB/NoPadding + + + http://git-wip-us.apache.org/repos/asf/accumulo/blob/e95dee9e/core/src/test/filtered-resources/crypto-on-no-key-encryption-accumulo-site.xml ---------------------------------------------------------------------- diff --git a/core/src/test/filtered-resources/crypto-on-no-key-encryption-accumulo-site.xml b/core/src/test/filtered-resources/crypto-on-no-key-encryption-accumulo-site.xml new file mode 100644 index 0000000..2fca01d --- /dev/null +++ b/core/src/test/filtered-resources/crypto-on-no-key-encryption-accumulo-site.xml @@ -0,0 +1,144 @@ + + + + + + + + + instance.zookeeper.host + localhost:2181 + comma separated list of zookeeper servers + + + + logger.dir.walog + walogs + The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories. + + + + instance.secret + DEFAULT + A secret unique to a given instance that all servers must know in order to communicate with one another. + Change it before initialization. To change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd], + and then update this file. + + + + + tserver.memory.maps.max + 80M + + + + tserver.cache.data.size + 7M + + + + tserver.cache.index.size + 20M + + + + trace.password + + password + + + + trace.user + root + + + + tserver.sort.buffer.size + 50M + + + + tserver.walog.max.size + 100M + + + + general.classpaths + + $ACCUMULO_HOME/server/target/classes/, + $ACCUMULO_HOME/core/target/classes/, + $ACCUMULO_HOME/start/target/classes/, + $ACCUMULO_HOME/fate/target/classes/, + $ACCUMULO_HOME/proxy/target/classes/, + $ACCUMULO_HOME/examples/target/classes/, + $ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar, + $ACCUMULO_HOME/lib/[^.].*.jar, + $ZOOKEEPER_HOME/zookeeper[^.].*.jar, + $HADOOP_CONF_DIR, + $HADOOP_PREFIX/[^.].*.jar, + $HADOOP_PREFIX/lib/[^.].*.jar, + + Classpaths that accumulo checks for updates and class files. + When using the Security Manager, please remove the ".../target/classes/" values. + + + + + crypto.module.class + org.apache.accumulo.core.security.crypto.DefaultCryptoModule + + + crypto.cipher.suite + AES/CFB/PKCS5Padding + + + crypto.cipher.algorithm.name + AES + + + crypto.cipher.key.length + 128 + + + crypto.secure.rng + SHA1PRNG + + + crypto.secure.rng.provider + SUN + + + instance.dfs.dir + ${project.build.directory} + + + instance.dfs.uri + file://${project.build.directory} + + + http://git-wip-us.apache.org/repos/asf/accumulo/blob/e95dee9e/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java index a61d4bb..e049531 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java @@ -68,100 +68,106 @@ import org.apache.hadoop.fs.Seekable; import org.apache.hadoop.io.Text; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; public class RFileTest { - + private static final Collection EMPTY_COL_FAMS = new ArrayList(); - + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target")); + static { Logger.getLogger(org.apache.hadoop.io.compress.CodecPool.class).setLevel(Level.OFF); Logger.getLogger(org.apache.hadoop.util.NativeCodeLoader.class).setLevel(Level.OFF); } - + static class SeekableByteArrayInputStream extends ByteArrayInputStream implements Seekable, PositionedReadable { - + public SeekableByteArrayInputStream(byte[] buf) { super(buf); } - + @Override public long getPos() throws IOException { return pos; } - + @Override public void seek(long pos) throws IOException { if (mark != 0) throw new IllegalStateException(); - + reset(); long skipped = skip(pos); - + if (skipped != pos) throw new IOException(); } - + @Override public boolean seekToNewSource(long targetPos) throws IOException { return false; } - + @Override public int read(long position, byte[] buffer, int offset, int length) throws IOException { - + if (position >= buf.length) throw new IllegalArgumentException(); if (position + length > buf.length) throw new IllegalArgumentException(); if (length > buffer.length) throw new IllegalArgumentException(); - + System.arraycopy(buf, (int) position, buffer, offset, length); return length; } - + @Override public void readFully(long position, byte[] buffer) throws IOException { read(position, buffer, 0, buffer.length); - + } - + @Override public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { read(position, buffer, offset, length); } - + } - + private static void checkIndex(Reader reader) throws IOException { FileSKVIterator indexIter = reader.getIndex(); - + if (indexIter.hasTop()) { Key lastKey = new Key(indexIter.getTopKey()); - + if (reader.getFirstKey().compareTo(lastKey) > 0) throw new RuntimeException("First key out of order " + reader.getFirstKey() + " " + lastKey); - + indexIter.next(); - + while (indexIter.hasTop()) { if (lastKey.compareTo(indexIter.getTopKey()) > 0) throw new RuntimeException("Indext out of order " + lastKey + " " + indexIter.getTopKey()); - + lastKey = new Key(indexIter.getTopKey()); indexIter.next(); - + } - + if (!reader.getLastKey().equals(lastKey)) { throw new RuntimeException("Last key out of order " + reader.getLastKey() + " " + lastKey); } } } - + public static class TestRFile { - + public File preGeneratedInputFile = null; public File outputFile = null; private Configuration conf = CachedConfiguration.getInstance(); @@ -172,12 +178,12 @@ public class RFileTest { private FSDataInputStream in; public Reader reader; public SortedKeyValueIterator iter; - + public void openWriter(boolean startDLG) throws IOException { - + if (outputFile == null) { baos = new ByteArrayOutputStream(); - + dos = new FSDataOutputStream(baos, new FileSystem.Statistics("a")); } else { BufferedOutputStream bufos = new BufferedOutputStream(new FileOutputStream(outputFile)); @@ -185,15 +191,15 @@ public class RFileTest { } CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(dos, "gz", conf); writer = new RFile.Writer(_cbw, 1000, 1000); - + if (startDLG) writer.startDefaultLocalityGroup(); } - + public void openWriter() throws IOException { openWriter(true); } - + public void closeWriter() throws IOException { dos.flush(); writer.close(); @@ -202,9 +208,9 @@ public class RFileTest { baos.close(); } } - + public void openReader() throws IOException { - + int fileLength = 0; byte[] data = null; if (preGeneratedInputFile != null) { @@ -215,73 +221,73 @@ public class RFileTest { } else { data = baos.toByteArray(); } - + bais = new SeekableByteArrayInputStream(data); in = new FSDataInputStream(bais); fileLength = data.length; - + LruBlockCache indexCache = new LruBlockCache(100000000, 100000); LruBlockCache dataCache = new LruBlockCache(100000000, 100000); - + CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(in, fileLength, conf, dataCache, indexCache); reader = new RFile.Reader(_cbr); iter = new ColumnFamilySkippingIterator(reader); - + checkIndex(reader); } - + public void closeReader() throws IOException { reader.close(); in.close(); } - + public void seek(Key nk) throws IOException { iter.seek(new Range(nk, null), EMPTY_COL_FAMS, false); } } - + static Key nk(String row, String cf, String cq, String cv, long ts) { return new Key(row.getBytes(), cf.getBytes(), cq.getBytes(), cv.getBytes(), ts); } - + static Value nv(String val) { return new Value(val.getBytes()); } - + static String nf(String prefix, int i) { return String.format(prefix + "%06d", i); } - + @Test public void test1() throws IOException { - + // test an empty file - + TestRFile trf = new TestRFile(); - + trf.openWriter(); trf.closeWriter(); - + trf.openReader(); trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false); assertFalse(trf.iter.hasTop()); - + assertEquals(null, trf.reader.getLastKey()); - + trf.closeReader(); } - + @Test public void test2() throws IOException { - + // test an rfile with one entry - + TestRFile trf = new TestRFile(); - + trf.openWriter(); trf.writer.append(nk("r1", "cf1", "cq1", "L1", 55), nv("foo")); trf.closeWriter(); - + trf.openReader(); // seek before everything trf.seek(null); @@ -290,11 +296,11 @@ public class RFileTest { assertTrue(trf.iter.getTopValue().equals(nv("foo"))); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + // seek after the key trf.seek(nk("r2", "cf1", "cq1", "L1", 55)); assertFalse(trf.iter.hasTop()); - + // seek exactly to the key trf.seek(nk("r1", "cf1", "cq1", "L1", 55)); assertTrue(trf.iter.hasTop()); @@ -302,26 +308,26 @@ public class RFileTest { assertTrue(trf.iter.getTopValue().equals(nv("foo"))); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + assertEquals(nk("r1", "cf1", "cq1", "L1", 55), trf.reader.getLastKey()); - + trf.closeReader(); } - + @Test public void test3() throws IOException { - + // test an rfile with multiple rows having multiple columns - + TestRFile trf = new TestRFile(); - + trf.openWriter(); - + int val = 0; - + ArrayList expectedKeys = new ArrayList(10000); ArrayList expectedValues = new ArrayList(10000); - + for (int row = 0; row < 4; row++) { String rowS = nf("r_", row); for (int cf = 0; cf < 4; cf++) { @@ -337,67 +343,67 @@ public class RFileTest { trf.writer.append(k, v); expectedKeys.add(k); expectedValues.add(v); - + k = nk(rowS, cfS, cqS, cvS, ts); v = nv("" + val); trf.writer.append(k, v); expectedKeys.add(k); expectedValues.add(v); - + val++; } } } } } - + // trf.writer.append(nk("r1","cf1","cq1","L1", 55), nv("foo")); trf.closeWriter(); - + trf.openReader(); // seek before everything trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false); verify(trf, expectedKeys.iterator(), expectedValues.iterator()); - + // seek to the middle int index = expectedKeys.size() / 2; trf.seek(expectedKeys.get(index)); verify(trf, expectedKeys.subList(index, expectedKeys.size()).iterator(), expectedValues.subList(index, expectedKeys.size()).iterator()); - + // seek the first key index = 0; trf.seek(expectedKeys.get(index)); verify(trf, expectedKeys.subList(index, expectedKeys.size()).iterator(), expectedValues.subList(index, expectedKeys.size()).iterator()); - + // seek to the last key index = expectedKeys.size() - 1; trf.seek(expectedKeys.get(index)); verify(trf, expectedKeys.subList(index, expectedKeys.size()).iterator(), expectedValues.subList(index, expectedKeys.size()).iterator()); - + // seek after everything index = expectedKeys.size(); trf.seek(new Key(new Text("z"))); verify(trf, expectedKeys.subList(index, expectedKeys.size()).iterator(), expectedValues.subList(index, expectedKeys.size()).iterator()); - + // test seeking to the current location index = expectedKeys.size() / 2; trf.seek(expectedKeys.get(index)); assertTrue(trf.iter.hasTop()); assertEquals(expectedKeys.get(index), trf.iter.getTopKey()); assertEquals(expectedValues.get(index), trf.iter.getTopValue()); - + trf.iter.next(); index++; assertTrue(trf.iter.hasTop()); assertEquals(expectedKeys.get(index), trf.iter.getTopKey()); assertEquals(expectedValues.get(index), trf.iter.getTopValue()); - + trf.seek(expectedKeys.get(index)); - + assertTrue(trf.iter.hasTop()); assertEquals(expectedKeys.get(index), trf.iter.getTopKey()); assertEquals(expectedValues.get(index), trf.iter.getTopValue()); - + // test seeking to each location in the file index = 0; for (Key key : expectedKeys) { @@ -405,34 +411,34 @@ public class RFileTest { assertTrue(trf.iter.hasTop()); assertEquals(key, trf.iter.getTopKey()); assertEquals(expectedValues.get(index), trf.iter.getTopValue()); - + if (index > 0) { // Key pkey = expectedKeys.get(index - 1); // assertEquals(pkey, trf.reader.getPrevKey()); } - + index++; } - + // test seeking backwards to each key for (int i = expectedKeys.size() - 1; i >= 0; i--) { Key key = expectedKeys.get(i); - + trf.seek(key); assertTrue(trf.iter.hasTop()); assertEquals(key, trf.iter.getTopKey()); assertEquals(expectedValues.get(i), trf.iter.getTopValue()); - + if (i - 1 > 0) { // Key pkey = expectedKeys.get(i - 1); // assertEquals(pkey, trf.reader.getPrevKey()); } } - + assertEquals(expectedKeys.get(expectedKeys.size() - 1), trf.reader.getLastKey()); - + // test seeking to random location and reading all data from that point // there was an off by one bug with this in the transient index Random rand = new Random(); @@ -446,112 +452,112 @@ public class RFileTest { trf.iter.next(); } } - + trf.closeReader(); } - + private void verify(TestRFile trf, Iterator eki, Iterator evi) throws IOException { - + while (trf.iter.hasTop()) { Key ek = eki.next(); Value ev = evi.next(); - + assertEquals(ek, trf.iter.getTopKey()); assertEquals(ev, trf.iter.getTopValue()); - + trf.iter.next(); } - + assertFalse(eki.hasNext()); assertFalse(evi.hasNext()); } - + @Test public void test4() throws IOException { TestRFile trf = new TestRFile(); - + trf.openWriter(); - + trf.writer.append(nk("r1", "cf1", "cq1", "L1", 55), nv("foo1")); try { trf.writer.append(nk("r0", "cf1", "cq1", "L1", 55), nv("foo1")); assertFalse(true); } catch (IllegalStateException ioe) { - + } - + try { trf.writer.append(nk("r1", "cf0", "cq1", "L1", 55), nv("foo1")); assertFalse(true); } catch (IllegalStateException ioe) { - + } - + try { trf.writer.append(nk("r1", "cf1", "cq0", "L1", 55), nv("foo1")); assertFalse(true); } catch (IllegalStateException ioe) { - + } - + try { trf.writer.append(nk("r1", "cf1", "cq1", "L0", 55), nv("foo1")); assertFalse(true); } catch (IllegalStateException ioe) { - + } - + try { trf.writer.append(nk("r1", "cf1", "cq1", "L1", 56), nv("foo1")); assertFalse(true); } catch (IllegalStateException ioe) { - + } } - + @Test public void test5() throws IOException { - + TestRFile trf = new TestRFile(); - + trf.openWriter(); trf.writer.append(nk("r1", "cf1", "cq1", "L1", 55), nv("foo1")); trf.writer.append(nk("r1", "cf1", "cq4", "L1", 56), nv("foo2")); trf.closeWriter(); - + trf.openReader(); - + // test seeking between keys trf.seek(nk("r1", "cf1", "cq3", "L1", 55)); assertTrue(trf.iter.hasTop()); assertEquals(nk("r1", "cf1", "cq4", "L1", 56), trf.iter.getTopKey()); assertEquals(nv("foo2"), trf.iter.getTopValue()); - + // test seeking right before previous seek trf.seek(nk("r1", "cf1", "cq0", "L1", 55)); assertTrue(trf.iter.hasTop()); assertEquals(nk("r1", "cf1", "cq1", "L1", 55), trf.iter.getTopKey()); assertEquals(nv("foo1"), trf.iter.getTopValue()); - + assertEquals(nk("r1", "cf1", "cq4", "L1", 56), trf.reader.getLastKey()); - + trf.closeReader(); } - + @Test public void test6() throws IOException { - + TestRFile trf = new TestRFile(); - + trf.openWriter(); for (int i = 0; i < 500; i++) { trf.writer.append(nk(nf("r_", i), "cf1", "cq1", "L1", 55), nv("foo1")); } - + trf.closeWriter(); - + trf.openReader(); - + // repeatedly seek to locations before the first key in the file for (int i = 0; i < 10; i++) { trf.seek(nk(nf("q_", i), "cf1", "cq1", "L1", 55)); @@ -559,33 +565,33 @@ public class RFileTest { assertEquals(nk(nf("r_", 0), "cf1", "cq1", "L1", 55), trf.iter.getTopKey()); assertEquals(nv("foo1"), trf.iter.getTopValue()); } - + // repeatedly seek to locations after the last key in the file for (int i = 0; i < 10; i++) { trf.seek(nk(nf("s_", i), "cf1", "cq1", "L1", 55)); assertFalse(trf.iter.hasTop()); } - + assertEquals(nk(nf("r_", 499), "cf1", "cq1", "L1", 55), trf.reader.getLastKey()); - + trf.closeReader(); } - + @Test public void test7() throws IOException { // these tests exercise setting the end key of a range - + TestRFile trf = new TestRFile(); - + trf.openWriter(); for (int i = 2; i < 50; i++) { trf.writer.append(nk(nf("r_", i), "cf1", "cq1", "L1", 55), nv("foo" + i)); } - + trf.closeWriter(); - + trf.openReader(); - + // test that has top returns false when end of range reached trf.iter.seek(new Range(nk(nf("r_", 3), "cf1", "cq1", "L1", 55), true, nk(nf("r_", 4), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false); assertTrue(trf.iter.hasTop()); @@ -593,19 +599,19 @@ public class RFileTest { assertEquals(nv("foo" + 3), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + // test seeking to a range that is between two keys, should not return anything trf.iter.seek(new Range(nk(nf("r_", 4) + "a", "cf1", "cq1", "L1", 55), true, nk(nf("r_", 4) + "b", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false); assertFalse(trf.iter.hasTop()); - + // test seeking to another range after the previously seeked range, that is between the same two keys in the file // as the previously seeked range.... this test an optimization on RFile trf.iter.seek(new Range(nk(nf("r_", 4) + "c", "cf1", "cq1", "L1", 55), true, nk(nf("r_", 4) + "d", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false); assertFalse(trf.iter.hasTop()); - + trf.iter.seek(new Range(nk(nf("r_", 4) + "e", "cf1", "cq1", "L1", 55), true, nk(nf("r_", 4) + "f", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false); assertFalse(trf.iter.hasTop()); - + // now ensure we can seek somewhere, that triggering the optimization does not cause any problems trf.iter.seek(new Range(nk(nf("r_", 5), "cf1", "cq1", "L1", 55), true, nk(nf("r_", 6), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false); assertTrue(trf.iter.hasTop()); @@ -613,158 +619,158 @@ public class RFileTest { assertEquals(nv("foo" + 5), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + // test seeking to range that is before the beginning of the file trf.iter.seek(new Range(nk(nf("r_", 0), "cf1", "cq1", "L1", 55), true, nk(nf("r_", 2), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false); assertFalse(trf.iter.hasTop()); - + assertEquals(nk(nf("r_", 49), "cf1", "cq1", "L1", 55), trf.reader.getLastKey()); - + trf.reader.close(); } - + @Test public void test8() throws IOException { TestRFile trf = new TestRFile(); - + trf.openWriter(); - + for (int i = 0; i < 2500; i++) { trf.writer.append(nk(nf("r_", i), "cf1", "cq1", "L1", 42), nv("foo" + i)); } - + trf.closeWriter(); trf.openReader(); - + // test seeking between each key forward for (int i = 0; i < 2499; i++) { trf.seek(nk(nf("r_", i), "cf1", "cq1", "L1", 42).followingKey(PartialKey.ROW)); assertTrue(trf.iter.hasTop()); assertEquals(nk(nf("r_", i + 1), "cf1", "cq1", "L1", 42), trf.iter.getTopKey()); } - + // test seeking between each key forward for (int i = 0; i < 2499; i += 2) { trf.seek(nk(nf("r_", i), "cf1", "cq1", "L1", 42).followingKey(PartialKey.ROW)); assertTrue(trf.iter.hasTop()); assertEquals(nk(nf("r_", i + 1), "cf1", "cq1", "L1", 42), trf.iter.getTopKey()); } - + // test seeking backwards between each key for (int i = 2498; i >= 0; i--) { trf.seek(nk(nf("r_", i), "cf1", "cq1", "L1", 42).followingKey(PartialKey.ROW)); assertTrue(trf.iter.hasTop()); assertEquals(nk(nf("r_", i + 1), "cf1", "cq1", "L1", 42), trf.iter.getTopKey()); } - + trf.closeReader(); - + // do same test with col fam trf = new TestRFile(); - + trf.openWriter(); - + for (int i = 0; i < 2500; i++) { trf.writer.append(nk(nf("r_", 0), nf("cf_", i), "cq1", "L1", 42), nv("foo" + i)); } - + trf.closeWriter(); trf.openReader(); - + // test seeking between each key forward for (int i = 0; i < 2499; i++) { trf.seek(nk(nf("r_", 0), nf("cf_", i), "cq1", "L1", 42).followingKey(PartialKey.ROW_COLFAM)); assertTrue(trf.iter.hasTop()); assertEquals(nk(nf("r_", 0), nf("cf_", i + 1), "cq1", "L1", 42), trf.iter.getTopKey()); } - + // test seeking between each key forward for (int i = 0; i < 2499; i += 2) { trf.seek(nk(nf("r_", 0), nf("cf_", i), "cq1", "L1", 42).followingKey(PartialKey.ROW_COLFAM)); assertTrue(trf.iter.hasTop()); assertEquals(nk(nf("r_", 0), nf("cf_", i + 1), "cq1", "L1", 42), trf.iter.getTopKey()); } - + // test seeking backwards between each key for (int i = 2498; i >= 0; i--) { trf.seek(nk(nf("r_", 0), nf("cf_", i), "cq1", "L1", 42).followingKey(PartialKey.ROW_COLFAM)); assertTrue(trf.iter.hasTop()); assertEquals(nk(nf("r_", 0), nf("cf_", i + 1), "cq1", "L1", 42), trf.iter.getTopKey()); } - + trf.closeReader(); - + // do same test with col qual trf = new TestRFile(); - + trf.openWriter(); - + for (int i = 0; i < 2500; i++) { trf.writer.append(nk(nf("r_", 0), nf("cf_", 0), nf("cq_", i), "L1", 42), nv("foo" + i)); } - + trf.closeWriter(); trf.openReader(); - + // test seeking between each key forward for (int i = 0; i < 2499; i++) { trf.seek(nk(nf("r_", 0), nf("cf_", 0), nf("cq_", i), "L1", 42).followingKey(PartialKey.ROW_COLFAM_COLQUAL)); assertTrue(trf.iter.hasTop()); assertEquals(nk(nf("r_", 0), nf("cf_", 0), nf("cq_", i + 1), "L1", 42), trf.iter.getTopKey()); } - + // test seeking between each key forward for (int i = 0; i < 2499; i += 2) { trf.seek(nk(nf("r_", 0), nf("cf_", 0), nf("cq_", i), "L1", 42).followingKey(PartialKey.ROW_COLFAM_COLQUAL)); assertTrue(trf.iter.hasTop()); assertEquals(nk(nf("r_", 0), nf("cf_", 0), nf("cq_", i + 1), "L1", 42), trf.iter.getTopKey()); } - + // test seeking backwards between each key for (int i = 2498; i >= 0; i--) { trf.seek(nk(nf("r_", 0), nf("cf_", 0), nf("cq_", i), "L1", 42).followingKey(PartialKey.ROW_COLFAM_COLQUAL)); assertTrue(trf.iter.hasTop()); assertEquals(nk(nf("r_", 0), nf("cf_", 0), nf("cq_", i + 1), "L1", 42), trf.iter.getTopKey()); } - + trf.closeReader(); } - + public static Set ncfs(String... colFams) { HashSet cfs = new HashSet(); - + for (String cf : colFams) { cfs.add(new ArrayByteSequence(cf)); } - + return cfs; } - + @Test public void test9() throws IOException { TestRFile trf = new TestRFile(); - + trf.openWriter(false); - + trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); - + trf.writer.append(nk("0000", "cf1", "doe,john", "", 4), nv("1123 West Left st")); trf.writer.append(nk("0002", "cf2", "doe,jane", "", 5), nv("1124 East Right st")); - + trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4")); - + trf.writer.append(nk("0001", "cf3", "buck,john", "", 4), nv("90 Slum st")); trf.writer.append(nk("0003", "cf4", "buck,jane", "", 5), nv("09 Slum st")); - + trf.writer.close(); - + trf.openReader(); - + // scan first loc group Range r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); trf.iter.seek(r, ncfs("cf1", "cf2"), true); assertEquals(1, trf.reader.getNumLocalityGroupsSeeked()); - + assertTrue(trf.iter.hasTop()); assertTrue(trf.iter.getTopKey().equals(nk("0000", "cf1", "doe,john", "", 4))); assertEquals(nv("1123 West Left st"), trf.iter.getTopValue()); @@ -774,12 +780,12 @@ public class RFileTest { assertEquals(nv("1124 East Right st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + // scan second loc group r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); trf.iter.seek(r, ncfs("cf3", "cf4"), true); assertEquals(1, trf.reader.getNumLocalityGroupsSeeked()); - + assertTrue(trf.iter.hasTop()); assertTrue(trf.iter.getTopKey().equals(nk("0001", "cf3", "buck,john", "", 4))); assertEquals(nv("90 Slum st"), trf.iter.getTopValue()); @@ -789,12 +795,12 @@ public class RFileTest { assertEquals(nv("09 Slum st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + // scan all loc groups r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); trf.iter.seek(r, EMPTY_COL_FAMS, false); assertEquals(2, trf.reader.getNumLocalityGroupsSeeked()); - + assertTrue(trf.iter.hasTop()); assertTrue(trf.iter.getTopKey().equals(nk("0000", "cf1", "doe,john", "", 4))); assertEquals(nv("1123 West Left st"), trf.iter.getTopValue()); @@ -812,62 +818,62 @@ public class RFileTest { assertEquals(nv("09 Slum st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + // scan no loc groups r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); trf.iter.seek(r, ncfs("saint", "dogooder"), true); assertEquals(0, trf.reader.getNumLocalityGroupsSeeked()); assertFalse(trf.iter.hasTop()); - + // scan a subset of second locality group r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); trf.iter.seek(r, ncfs("cf4"), true); assertEquals(1, trf.reader.getNumLocalityGroupsSeeked()); - + assertTrue(trf.iter.hasTop()); assertTrue(trf.iter.getTopKey().equals(nk("0003", "cf4", "buck,jane", "", 5))); assertEquals(nv("09 Slum st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + // scan a subset of second locality group r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); trf.iter.seek(r, ncfs("cf3"), true); assertEquals(1, trf.reader.getNumLocalityGroupsSeeked()); - + assertTrue(trf.iter.hasTop()); assertTrue(trf.iter.getTopKey().equals(nk("0001", "cf3", "buck,john", "", 4))); assertEquals(nv("90 Slum st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + // scan subset of first loc group r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); trf.iter.seek(r, ncfs("cf1"), true); assertEquals(1, trf.reader.getNumLocalityGroupsSeeked()); - + assertTrue(trf.iter.hasTop()); assertTrue(trf.iter.getTopKey().equals(nk("0000", "cf1", "doe,john", "", 4))); assertEquals(nv("1123 West Left st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + // scan subset of first loc group r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); trf.iter.seek(r, ncfs("cf2"), true); assertEquals(1, trf.reader.getNumLocalityGroupsSeeked()); - + assertTrue(trf.iter.hasTop()); assertTrue(trf.iter.getTopKey().equals(nk("0002", "cf2", "doe,jane", "", 5))); assertEquals(nv("1124 East Right st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + // scan subset of all loc groups r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true); trf.iter.seek(r, ncfs("cf1", "cf4"), true); assertEquals(2, trf.reader.getNumLocalityGroupsSeeked()); - + assertTrue(trf.iter.hasTop()); assertTrue(trf.iter.getTopKey().equals(nk("0000", "cf1", "doe,john", "", 4))); assertEquals(nv("1123 West Left st"), trf.iter.getTopValue()); @@ -877,32 +883,32 @@ public class RFileTest { assertEquals(nv("09 Slum st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + trf.closeReader(); - + } - + @Test public void test10() throws IOException { - + // test empty locality groups TestRFile trf = new TestRFile(); - + trf.openWriter(false); trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4")); trf.writer.startDefaultLocalityGroup(); trf.writer.close(); - + trf.openReader(); trf.iter.seek(new Range(new Text(""), null), EMPTY_COL_FAMS, false); assertFalse(trf.iter.hasTop()); - + trf.closeReader(); - + // another empty locality group test trf = new TestRFile(); - + trf.openWriter(false); trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); trf.writer.append(nk("0000", "cf1", "doe,john", "", 4), nv("1123 West Left st")); @@ -910,7 +916,7 @@ public class RFileTest { trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4")); trf.writer.startDefaultLocalityGroup(); trf.writer.close(); - + trf.openReader(); trf.iter.seek(new Range(new Text(""), null), EMPTY_COL_FAMS, false); assertTrue(trf.iter.hasTop()); @@ -922,12 +928,12 @@ public class RFileTest { assertEquals(nv("1124 East Right st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + trf.closeReader(); - + // another empty locality group test trf = new TestRFile(); - + trf.openWriter(false); trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4")); @@ -935,7 +941,7 @@ public class RFileTest { trf.writer.append(nk("0003", "cf4", "buck,jane", "", 5), nv("09 Slum st")); trf.writer.startDefaultLocalityGroup(); trf.writer.close(); - + trf.openReader(); trf.iter.seek(new Range(new Text(""), null), EMPTY_COL_FAMS, false); assertTrue(trf.iter.hasTop()); @@ -947,12 +953,12 @@ public class RFileTest { assertEquals(nv("09 Slum st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + trf.closeReader(); - + // another empty locality group test trf = new TestRFile(); - + trf.openWriter(false); trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4")); @@ -960,7 +966,7 @@ public class RFileTest { trf.writer.append(nk("0007", "good citizen", "q,john", "", 4), nv("70 Apple st")); trf.writer.append(nk("0008", "model citizen", "q,jane", "", 5), nv("81 Plum st")); trf.writer.close(); - + trf.openReader(); trf.iter.seek(new Range(new Text(""), null), EMPTY_COL_FAMS, false); assertTrue(trf.iter.hasTop()); @@ -972,12 +978,12 @@ public class RFileTest { assertEquals(nv("81 Plum st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + trf.closeReader(); - + // another empty locality group test trf = new TestRFile(); - + trf.openWriter(false); trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); trf.writer.append(nk("0000", "cf1", "doe,john", "", 4), nv("1123 West Left st")); @@ -987,7 +993,7 @@ public class RFileTest { trf.writer.append(nk("0007", "good citizen", "q,john", "", 4), nv("70 Apple st")); trf.writer.append(nk("0008", "model citizen", "q,jane", "", 5), nv("81 Plum st")); trf.writer.close(); - + trf.openReader(); trf.iter.seek(new Range(new Text(""), null), EMPTY_COL_FAMS, false); assertTrue(trf.iter.hasTop()); @@ -1007,43 +1013,43 @@ public class RFileTest { assertEquals(nv("81 Plum st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + trf.closeReader(); } - + @Test public void test11() throws IOException { // test locality groups with more than two entries - + TestRFile trf = new TestRFile(); - + trf.openWriter(false); trf.writer.startNewLocalityGroup("lg1", ncfs("3mod10")); for (int i = 3; i < 1024; i += 10) { trf.writer.append(nk(nf("i", i), "3mod10", "", "", i + 2), nv("" + i)); } - + trf.writer.startNewLocalityGroup("lg2", ncfs("5mod10", "7mod10")); - + for (int i = 5; i < 1024;) { trf.writer.append(nk(nf("i", i), "5mod10", "", "", i + 2), nv("" + i)); i += 2; trf.writer.append(nk(nf("i", i), "7mod10", "", "", i + 2), nv("" + i)); i += 8; } - + trf.writer.startDefaultLocalityGroup(); for (int i = 0; i < 1024; i++) { - + int m10 = i % 10; if (m10 == 3 || m10 == 5 || m10 == 7) continue; - + trf.writer.append(nk(nf("i", i), m10 + "mod10", "", "", i + 2), nv("" + i)); - + } trf.writer.close(); - + // test a merged read of all column families trf.openReader(); trf.iter.seek(new Range(new Text(""), null), EMPTY_COL_FAMS, false); @@ -1055,7 +1061,7 @@ public class RFileTest { trf.iter.next(); } assertFalse(trf.iter.hasTop()); - + // try reading each of the 10 column families separately for (int m = 0; m < 10; m++) { trf.iter.seek(new Range(new Key(), true, null, true), ncfs(m + "mod10"), true); @@ -1067,7 +1073,7 @@ public class RFileTest { trf.iter.next(); } assertFalse(trf.iter.hasTop()); - + // test excluding an individual column family trf.iter.seek(new Range(new Key(), true, null, true), ncfs(m + "mod10"), false); if (m == 3) @@ -1075,10 +1081,10 @@ public class RFileTest { else assertEquals(3, trf.reader.getNumLocalityGroupsSeeked()); for (int i = 0; i < 1024; i++) { - + if (i % 10 == m) continue; - + assertTrue(trf.iter.hasTop()); assertEquals(nk(nf("i", i), (i % 10) + "mod10", "", "", i + 2), trf.iter.getTopKey()); assertEquals(nv("" + i), trf.iter.getTopValue()); @@ -1086,10 +1092,10 @@ public class RFileTest { } assertFalse(trf.iter.hasTop()); } - + // test Rfile deepcopy SortedKeyValueIterator reader2 = trf.iter.deepCopy(null); - + // try reading from cloned reader at the same time as parent reader for (int m = 0; m < 9; m++) { trf.iter.seek(new Range(new Key(), true, null, true), ncfs(m + "mod10"), true); @@ -1112,441 +1118,441 @@ public class RFileTest { assertFalse(trf.iter.hasTop()); assertFalse(reader2.hasTop()); } - + trf.closeReader(); } - + @Test public void test12() throws IOException { // test inserting column fams not in locality groups - + TestRFile trf = new TestRFile(); - + trf.openWriter(false); - + trf.writer.startNewLocalityGroup("lg1", ncfs("a", "b")); - + trf.writer.append(nk("0007", "a", "cq1", "", 4), nv("1")); - + try { trf.writer.append(nk("0009", "c", "cq1", "", 4), nv("1")); assertFalse(true); } catch (IllegalArgumentException ioe) { - + } - + trf.closeWriter(); - + trf.openReader(); - + trf.iter.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(trf.iter.hasTop()); assertEquals(nk("0007", "a", "cq1", "", 4), trf.iter.getTopKey()); assertEquals(nv("1"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + } - + @Test public void test13() throws IOException { // test inserting column fam in default loc group that was in // previous locality group - + TestRFile trf = new TestRFile(); - + trf.openWriter(false); - + trf.writer.startNewLocalityGroup("lg1", ncfs("a", "b")); - + trf.writer.append(nk("0007", "a", "cq1", "", 4), nv("1")); - + trf.writer.startDefaultLocalityGroup(); - + try { trf.writer.append(nk("0008", "a", "cq1", "", 4), nv("1")); assertFalse(true); } catch (IllegalArgumentException ioe) { - + } - + try { trf.writer.append(nk("0009", "b", "cq1", "", 4), nv("1")); assertFalse(true); } catch (IllegalArgumentException ioe) { - + } - + trf.closeWriter(); - + trf.openReader(); - + trf.iter.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(trf.iter.hasTop()); assertEquals(nk("0007", "a", "cq1", "", 4), trf.iter.getTopKey()); assertEquals(nv("1"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + } - + @Test public void test14() throws IOException { // test starting locality group after default locality group was started - + TestRFile trf = new TestRFile(); - + trf.openWriter(false); - + trf.writer.startDefaultLocalityGroup(); try { trf.writer.startNewLocalityGroup("lg1", ncfs("a", "b")); assertFalse(true); } catch (IllegalStateException ioe) { - + } - + try { trf.writer.startDefaultLocalityGroup(); assertFalse(true); } catch (IllegalStateException ioe) { - + } - + trf.writer.close(); } - + @Test public void test16() throws IOException { TestRFile trf = new TestRFile(); - + trf.openWriter(false); - + trf.writer.startNewLocalityGroup("lg1", ncfs("a", "b")); - + trf.writer.append(nk("0007", "a", "cq1", "", 4), nv("1")); try { trf.writer.startNewLocalityGroup("lg1", ncfs("b", "c")); assertFalse(true); } catch (IllegalArgumentException ioe) { - + } - + trf.closeWriter(); } - + @Test public void test17() throws IOException { // add alot of the same keys to rfile that cover multiple blocks... // this should cause the keys in the index to be exactly the same... // ensure seeks work correctly - + TestRFile trf = new TestRFile(); - + trf.openWriter(false); - + trf.writer.startDefaultLocalityGroup(); for (int i = 0; i < 2048; i++) { trf.writer.append(nk("r0000", "cf1", "cq1", "", 1), nv("" + i)); } - + for (int i = 2048; i < 4096; i++) { trf.writer.append(nk("r0001", "cf1", "cq1", "", 1), nv("" + i)); } - + trf.writer.close(); - + trf.openReader(); - + FileSKVIterator indexIter = trf.reader.getIndex(); int count = 0; while (indexIter.hasTop()) { count++; indexIter.next(); } - + assertTrue(count > 4); - + trf.iter.seek(new Range(nk("r0000", "cf1", "cq1", "", 1), true, nk("r0001", "cf1", "cq1", "", 1), false), EMPTY_COL_FAMS, false); - + for (int i = 0; i < 2048; i++) { assertTrue(trf.iter.hasTop()); assertEquals(nk("r0000", "cf1", "cq1", "", 1), trf.iter.getTopKey()); assertEquals(nv("" + i), trf.iter.getTopValue()); trf.iter.next(); } - + assertFalse(trf.iter.hasTop()); - + trf.iter.seek(new Range(nk("r0000", "cf1", "cq1", "", 1), false, nk("r0001", "cf1", "cq1", "", 1), true), EMPTY_COL_FAMS, false); - + for (int i = 2048; i < 4096; i++) { assertTrue(trf.iter.hasTop()); assertEquals(nk("r0001", "cf1", "cq1", "", 1), trf.iter.getTopKey()); assertEquals(nv("" + i), trf.iter.getTopValue()); trf.iter.next(); } - + assertFalse(trf.iter.hasTop()); - + trf.iter.seek(new Range(nk("r0001", "cf1", "cq1", "", 1), true, nk("r0001", "cf1", "cq1", "", 1), true), EMPTY_COL_FAMS, false); - + for (int i = 2048; i < 4096; i++) { assertTrue(trf.iter.hasTop()); assertEquals(nk("r0001", "cf1", "cq1", "", 1), trf.iter.getTopKey()); assertEquals(nv("" + i), trf.iter.getTopValue()); trf.iter.next(); } - + assertFalse(trf.iter.hasTop()); - + trf.iter.seek(new Range(nk("r0002", "cf1", "cq1", "", 1), true, nk("r0002", "cf1", "cq1", "", 1), true), EMPTY_COL_FAMS, false); assertFalse(trf.iter.hasTop()); - + trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false); - + for (int i = 0; i < 2048; i++) { assertTrue(trf.iter.hasTop()); assertEquals(nk("r0000", "cf1", "cq1", "", 1), trf.iter.getTopKey()); assertEquals(nv("" + i), trf.iter.getTopValue()); trf.iter.next(); } - + for (int i = 2048; i < 4096; i++) { assertTrue(trf.iter.hasTop()); assertEquals(nk("r0001", "cf1", "cq1", "", 1), trf.iter.getTopKey()); assertEquals(nv("" + i), trf.iter.getTopValue()); trf.iter.next(); } - + assertFalse(trf.iter.hasTop()); - + trf.closeReader(); } - + private String t18ncf(int i) { return String.format("cf%06d", i); } - + private Set t18ncfs(int... colFams) { HashSet cfs = new HashSet(); for (int i : colFams) { cfs.add(new ArrayByteSequence(t18ncf(i))); } - + return cfs; } - + private void t18Append(TestRFile trf, HashSet allCf, int i) throws IOException { String cf = t18ncf(i); trf.writer.append(nk("r0000", cf, "cq1", "", 1), nv("" + i)); allCf.add(new ArrayByteSequence(cf)); } - + private void t18Verify(Set cfs, SortedKeyValueIterator iter, Reader reader, HashSet allCf, int eialg, int eealg) throws IOException { - + HashSet colFamsSeen = new HashSet(); - + iter.seek(new Range(), cfs, true); assertEquals(eialg, reader.getNumLocalityGroupsSeeked()); - + while (iter.hasTop()) { colFamsSeen.add(iter.getTopKey().getColumnFamilyData()); iter.next(); } - + HashSet expected = new HashSet(allCf); expected.retainAll(cfs); assertEquals(expected, colFamsSeen); - + iter.seek(new Range(), cfs, false); assertEquals(eealg, reader.getNumLocalityGroupsSeeked()); - + colFamsSeen.clear(); while (iter.hasTop()) { colFamsSeen.add(iter.getTopKey().getColumnFamilyData()); iter.next(); } - + HashSet nonExcluded = new HashSet(allCf); nonExcluded.removeAll(cfs); assertEquals(nonExcluded, colFamsSeen); } - + @Test public void test18() throws IOException { // test writing more column families to default LG than it will track - + TestRFile trf = new TestRFile(); - + trf.openWriter(false); - + HashSet allCf = new HashSet(); - + trf.writer.startNewLocalityGroup("lg1", t18ncfs(0)); for (int i = 0; i < 1; i++) t18Append(trf, allCf, i); - + trf.writer.startNewLocalityGroup("lg2", t18ncfs(1, 2)); for (int i = 1; i < 3; i++) t18Append(trf, allCf, i); - + trf.writer.startNewLocalityGroup("lg3", t18ncfs(3, 4, 5)); for (int i = 3; i < 6; i++) t18Append(trf, allCf, i); - + trf.writer.startDefaultLocalityGroup(); - + int max = 6 + RFile.Writer.MAX_CF_IN_DLG + 100; for (int i = 6; i < max; i++) t18Append(trf, allCf, i); - + trf.closeWriter(); - + trf.openReader(); - + t18Verify(t18ncfs(0), trf.iter, trf.reader, allCf, 1, 3); for (int i = 1; i < 10; i++) t18Verify(t18ncfs(i), trf.iter, trf.reader, allCf, 1, 4); - + t18Verify(t18ncfs(max + 1), trf.iter, trf.reader, allCf, 1, 4); - + t18Verify(t18ncfs(1, 2, 3, 4), trf.iter, trf.reader, allCf, 2, 3); t18Verify(t18ncfs(1, 2, 3, 4, 5), trf.iter, trf.reader, allCf, 2, 2); - + t18Verify(t18ncfs(0, 1, 2, 3, 4), trf.iter, trf.reader, allCf, 3, 2); t18Verify(t18ncfs(0, 1, 2, 3, 4, 5), trf.iter, trf.reader, allCf, 3, 1); t18Verify(t18ncfs(0, 1, 2, 3, 4, 5, 6), trf.iter, trf.reader, allCf, 4, 1); - + t18Verify(t18ncfs(0, 1), trf.iter, trf.reader, allCf, 2, 3); t18Verify(t18ncfs(2, 3), trf.iter, trf.reader, allCf, 2, 4); t18Verify(t18ncfs(5, 6), trf.iter, trf.reader, allCf, 2, 4); - + trf.closeReader(); } - + @Test public void test19() throws IOException { // test RFile metastore TestRFile trf = new TestRFile(); - + trf.openWriter(false); - + trf.openWriter(false); trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2")); trf.writer.append(nk("0000", "cf1", "doe,john", "", 4), nv("1123 West Left st")); trf.writer.append(nk("0002", "cf2", "doe,jane", "", 5), nv("1124 East Right st")); trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4")); - + DataOutputStream dos = trf.writer.createMetaStore("count"); - + dos.writeInt(2); dos.writeUTF("data1"); dos.writeInt(1); dos.writeUTF("data2"); dos.writeInt(1); - + dos.close(); - + trf.closeWriter(); - + trf.openReader(); - + trf.iter.seek(new Range(), EMPTY_COL_FAMS, false); - + assertTrue(trf.iter.hasTop()); assertTrue(trf.iter.getTopKey().equals(nk("0000", "cf1", "doe,john", "", 4))); assertEquals(nv("1123 West Left st"), trf.iter.getTopValue()); trf.iter.next(); - + DataInputStream in = trf.reader.getMetaStore("count"); - + assertEquals(2, in.readInt()); assertEquals("data1", in.readUTF()); assertEquals(1, in.readInt()); assertEquals("data2", in.readUTF()); assertEquals(1, in.readInt()); - + in.close(); - + assertTrue(trf.iter.hasTop()); assertTrue(trf.iter.getTopKey().equals(nk("0002", "cf2", "doe,jane", "", 5))); assertEquals(nv("1124 East Right st"), trf.iter.getTopValue()); trf.iter.next(); assertFalse(trf.iter.hasTop()); - + trf.closeReader(); } - + @Test public void testReseekUnconsumed() throws Exception { TestRFile trf = new TestRFile(); - + trf.openWriter(); - + for (int i = 0; i < 2500; i++) { trf.writer.append(nk(nf("r_", i), "cf1", "cq1", "L1", 42), nv("foo" + i)); } - + trf.closeWriter(); trf.openReader(); - + Set cfs = Collections.emptySet(); - + Random rand = new Random(); - + for (int count = 0; count < 100; count++) { - + int start = rand.nextInt(2300); Range range = new Range(nk(nf("r_", start), "cf1", "cq1", "L1", 42), nk(nf("r_", start + 100), "cf1", "cq1", "L1", 42)); - + trf.reader.seek(range, cfs, false); - + int numToScan = rand.nextInt(100); - + for (int j = 0; j < numToScan; j++) { assertTrue(trf.reader.hasTop()); assertEquals(nk(nf("r_", start + j), "cf1", "cq1", "L1", 42), trf.reader.getTopKey()); trf.reader.next(); } - + assertTrue(trf.reader.hasTop()); assertEquals(nk(nf("r_", start + numToScan), "cf1", "cq1", "L1", 42), trf.reader.getTopKey()); - + // seek a little forward from the last range and read a few keys within the unconsumed portion of the last range - + int start2 = start + numToScan + rand.nextInt(3); int end2 = start2 + rand.nextInt(3); - + range = new Range(nk(nf("r_", start2), "cf1", "cq1", "L1", 42), nk(nf("r_", end2), "cf1", "cq1", "L1", 42)); trf.reader.seek(range, cfs, false); - + for (int j = start2; j <= end2; j++) { assertTrue(trf.reader.hasTop()); assertEquals(nk(nf("r_", j), "cf1", "cq1", "L1", 42), trf.reader.getTopKey()); trf.reader.next(); } - + assertFalse(trf.reader.hasTop()); - + } - + trf.closeReader(); } - + @Test(expected = NullPointerException.class) public void testMissingUnreleasedVersions() throws Exception { runVersionTest(5); } - + @Test public void testOldVersions() throws Exception { runVersionTest(3); runVersionTest(4); runVersionTest(6); } - + private void runVersionTest(int version) throws IOException { InputStream in = this.getClass().getClassLoader().getResourceAsStream("org/apache/accumulo/core/file/rfile/ver_" + version + ".rf"); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -1554,38 +1560,38 @@ public class RFileTest { int read; while ((read = in.read(buf)) > 0) baos.write(buf, 0, read); - + byte data[] = baos.toByteArray(); SeekableByteArrayInputStream bais = new SeekableByteArrayInputStream(data); FSDataInputStream in2 = new FSDataInputStream(bais); CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(in2, data.length, CachedConfiguration.getInstance()); Reader reader = new RFile.Reader(_cbr); checkIndex(reader); - + ColumnFamilySkippingIterator iter = new ColumnFamilySkippingIterator(reader); - + for (int start : new int[] {0, 10, 100, 998}) { for (int cf = 1; cf <= 4; cf++) { if (start == 0) iter.seek(new Range(), ncfs(nf("cf_", cf)), true); else iter.seek(new Range(nf("r_", start), null), ncfs(nf("cf_", cf)), true); - + for (int i = start; i < 1000; i++) { assertTrue(iter.hasTop()); assertEquals(nk(nf("r_", i), nf("cf_", cf), nf("cq_", 0), "", 1000 - i), iter.getTopKey()); assertEquals(nv(i + ""), iter.getTopValue()); iter.next(); } - + assertFalse(iter.hasTop()); } - + if (start == 0) iter.seek(new Range(), ncfs(), false); else iter.seek(new Range(nf("r_", start), null), ncfs(), false); - + for (int i = start; i < 1000; i++) { for (int cf = 1; cf <= 4; cf++) { assertTrue(iter.hasTop()); @@ -1594,13 +1600,13 @@ public class RFileTest { iter.next(); } } - + assertFalse(iter.hasTop()); } - + reader.close(); } - + private AccumuloConfiguration setAndGetAccumuloConfig(String cryptoConfSetting) { @SuppressWarnings("deprecation") AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration(); @@ -1608,7 +1614,7 @@ public class RFileTest { ((SiteConfiguration) conf).clearAndNull(); return conf; } - + private void restoreOldConfiguration(String oldSiteConfigProperty, AccumuloConfiguration conf) { if (oldSiteConfigProperty != null) { System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, oldSiteConfigProperty); @@ -1617,195 +1623,196 @@ public class RFileTest { } ((SiteConfiguration) conf).clearAndNull(); } - + @Test public void testEncRFile1() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test1(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile2() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test2(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile3() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test3(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile4() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test4(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile5() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test5(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile6() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test6(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile7() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test7(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile8() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test8(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile9() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test9(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile10() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test10(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile11() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test11(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile12() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test12(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile13() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test13(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile14() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test14(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile16() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test16(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile17() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test17(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile18() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test18(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - + @Test public void testEncRFile19() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); AccumuloConfiguration conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF); - + test19(); - + restoreOldConfiguration(oldSiteConfigProperty, conf); } - - // @Test + + @Test + @Ignore public void testEncryptedRFiles() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); @SuppressWarnings("deprecation") AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration(); System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_ON_CONF); ((SiteConfiguration) conf).clearAndNull(); - + test1(); test2(); test3(); @@ -1814,7 +1821,7 @@ public class RFileTest { test6(); test7(); test8(); - + if (oldSiteConfigProperty != null) { System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, oldSiteConfigProperty); } else { @@ -1822,136 +1829,137 @@ public class RFileTest { } ((SiteConfiguration) conf).clearAndNull(); } - - // @Test + + @Test + @Ignore public void testRootTabletFromServer() throws Exception { String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); @SuppressWarnings("deprecation") AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration(); System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_ON_CONF); ((SiteConfiguration) conf).clearAndNull(); - + TestRFile trf = new TestRFile(); - trf.preGeneratedInputFile = new File("/tmp/should_work.rf"); - + trf.preGeneratedInputFile = tempFolder.newFile("should_work.rf"); + trf.openReader(); trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false); assertTrue(trf.iter.hasTop()); - + assertTrue(trf.reader.getLastKey() != null); - + trf.closeReader(); - + if (oldSiteConfigProperty != null) { System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, oldSiteConfigProperty); } else { System.clearProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); } ((SiteConfiguration) conf).clearAndNull(); - + } - + private static final int TOTAL_NUM_ROWS = 10; private static final int ROW_MOD_VALUE = 10; - - // @Test + + @Test + @Ignore // These tests will purge the disk cache when the run, so it's not recommended that they be run in development systems. public void testEncryptedRFileWriteSpeed() throws Exception { - + System.out .println("Unencrypted Write, Unencrypted Read (Cache), Unencrypted Read (FS only), Encrypted Write, Encrypted Read (Cache), Encrypted Read (FS Only)"); int numIterations = 1; - + for (int i = 0; i < numIterations; i++) { @SuppressWarnings("deprecation") AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration(); System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_OFF_CONF); ((SiteConfiguration) conf).clearAndNull(); - + TestRFile trf = new TestRFile(); - trf.outputFile = new File("/tmp/testUnencryptedRfile.rf"); + trf.outputFile = tempFolder.newFile("testUnencryptedRfile.rf"); trf.openWriter(); - + double timeTickSize = 1000.0; int numRowsRead = 0; - + try { - + performUnencryptedTests(trf, TOTAL_NUM_ROWS, ROW_MOD_VALUE, timeTickSize, true); - + performEncryptedTests(TOTAL_NUM_ROWS, ROW_MOD_VALUE, timeTickSize, numRowsRead, false); - + } catch (Exception e) { e.printStackTrace(); throw e; } - + } - + } - + private void performUnencryptedTests(TestRFile trf, int totalNumRows, int rowModValue, double timeTickSize, boolean first) throws IOException, InterruptedException { long start = System.currentTimeMillis(); - + writeRowsToRfile(trf, totalNumRows, rowModValue); - + long end = System.currentTimeMillis(); - + System.out.print("" + ((end - start) / timeTickSize) + ", "); - + trf.preGeneratedInputFile = trf.outputFile; - + start = System.currentTimeMillis(); - + trf.openReader(); trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false); int numRowsRead = 0; - + int numRowsToRead = totalNumRows; while (numRowsRead < TOTAL_NUM_ROWS) { int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead); - + numRowsToRead -= numRowsReadThisTime; numRowsRead += numRowsReadThisTime; } - + trf.closeReader(); - + end = System.currentTimeMillis(); - + System.out.print("" + ((end - start) / timeTickSize) + ", "); - + Runtime.getRuntime().exec("purge").waitFor(); - + start = System.currentTimeMillis(); - + trf.openReader(); trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false); numRowsRead = 0; - + numRowsToRead = totalNumRows; while (numRowsRead < TOTAL_NUM_ROWS) { int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead); - + numRowsToRead -= numRowsReadThisTime; numRowsRead += numRowsReadThisTime; } - + trf.closeReader(); - + end = System.currentTimeMillis(); - + if (first) { System.out.print("" + ((end - start) / timeTickSize) + ", "); } else { System.out.println("" + ((end - start) / timeTickSize)); - + } - - // trf.outputFile.delete(); + } - + @SuppressWarnings("deprecation") private void performEncryptedTests(int totalNumRows, int rowModValue, double timeTickSize, int numRowsRead, boolean first) throws IOException, InterruptedException { @@ -1960,220 +1968,219 @@ public class RFileTest { long start; long end; int numRowsToRead; - + conf = AccumuloConfiguration.getSiteConfiguration(); System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_ON_CONF); ((SiteConfiguration) conf).clearAndNull(); - + trf = new TestRFile(); - trf.outputFile = new File("/tmp/testEncryptedRfile.rf"); + trf.outputFile = tempFolder.newFile("testEncryptedRfile.rf"); trf.openWriter(); - + start = System.currentTimeMillis(); - + writeRowsToRfile(trf, totalNumRows, rowModValue); - + end = System.currentTimeMillis(); - + System.out.print("" + ((end - start) / timeTickSize) + ", "); - + trf.preGeneratedInputFile = trf.outputFile; - + start = System.currentTimeMillis(); - + trf.openReader(); trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false); - + numRowsToRead = totalNumRows; while (numRowsRead < TOTAL_NUM_ROWS) { int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead); - + numRowsToRead -= numRowsReadThisTime; numRowsRead += numRowsReadThisTime; } - + trf.closeReader(); - + end = System.currentTimeMillis(); - + System.out.print("" + ((end - start) / timeTickSize) + ", "); - + Runtime.getRuntime().exec("purge").waitFor(); - + start = System.currentTimeMillis(); - + trf.openReader(); trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false); numRowsRead = 0; - + numRowsToRead = totalNumRows; while (numRowsRead < TOTAL_NUM_ROWS) { int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead); - + numRowsToRead -= numRowsReadThisTime; numRowsRead += numRowsReadThisTime; } - + trf.closeReader(); - + end = System.currentTimeMillis(); - + if (first) { System.out.print("" + ((end - start) / timeTickSize) + ", "); } else { System.out.println("" + ((end - start) / timeTickSize)); - + } - - trf.outputFile.delete(); + } - + private int readRandomRowsFromRfile(TestRFile trf, int totalRowCount, int maxRowsToRead) throws IOException { if (maxRowsToRead <= 0) { return 0; } - + int numRowsRead = 0; Random rand = new Random(System.nanoTime()); - + int firstKeyNum = Math.abs(rand.nextInt()) % totalRowCount; // int lastKeyNum = Math.abs(rand.nextInt()) % totalRowCount; int lastKeyNum = firstKeyNum + 1; - + if (lastKeyNum >= totalRowCount) { lastKeyNum = firstKeyNum; } - + if (lastKeyNum < firstKeyNum) { int temp = lastKeyNum; lastKeyNum = firstKeyNum; firstKeyNum = temp; } - + if (lastKeyNum - firstKeyNum > maxRowsToRead) { lastKeyNum = firstKeyNum + maxRowsToRead; } - + Key firstKey = nk(nf("r_", firstKeyNum), "cf_0", "cq_0", "vis", 0L); Key lastKey = nk(nf("r_", lastKeyNum), "cf_19", "cq_19", "vis", 0L); - + trf.iter.seek(new Range(firstKey, lastKey), EMPTY_COL_FAMS, false); for (int i = firstKeyNum; i < lastKeyNum; i++) { @SuppressWarnings("unused") Key k = trf.iter.getTopKey(); @SuppressWarnings("unused") Value v = trf.iter.getTopValue(); - + trf.iter.next(); - + numRowsRead++; } - + return numRowsRead; - + } - + private void writeRowsToRfile(TestRFile trf, int numRowsToWriteAndRead, int rowModValue) throws IOException { for (int i = 0; i < numRowsToWriteAndRead; i++) { String rowID = nf("r_", (i % rowModValue)); String colFam = nf("cf_", (i % 20)); String colQual = nf("cq_", (i % 20)); String colVis = "vis"; - + Key k = nk(rowID, colFam, colQual, colVis, i); Value v = nv("" + i); - + trf.writer.append(k, v); } - + trf.closeWriter(); } - + @Test public void testRootTabletEncryption() throws Exception { - + // This tests that the normal set of operations used to populate a root tablet - + String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); @SuppressWarnings("deprecation") AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration(); System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_ON_CONF); ((SiteConfiguration) conf).clearAndNull(); - + // populate the root tablet with info about the default tablet // the root tablet contains the key extent and locations of all the // metadata tablets // String initRootTabFile = ServerConstants.getMetadataTableDir() + "/root_tablet/00000_00000." // + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration()); // FileSKVWriter mfw = FileOperations.getInstance().openWriter(initRootTabFile, fs, conf, AccumuloConfiguration.getDefaultConfiguration()); - + TestRFile testRfile = new TestRFile(); testRfile.openWriter(); - + RFile.Writer mfw = testRfile.writer; - + // mfw.startDefaultLocalityGroup(); - + // mfw.startDefaultLocalityGroup(); - + Text tableExtent = new Text(KeyExtent.getMetadataEntry(new Text(MetadataTable.ID), MetadataSchema.TabletsSection.getRange().getEndKey().getRow())); - + // table tablet's directory Key tableDirKey = new Key(tableExtent, TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnFamily(), TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnQualifier(), 0); mfw.append(tableDirKey, new Value(/* TABLE_TABLETS_TABLET_DIR */"/table_info".getBytes())); - + // table tablet time Key tableTimeKey = new Key(tableExtent, TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnFamily(), TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnQualifier(), 0); mfw.append(tableTimeKey, new Value((/* TabletTime.LOGICAL_TIME_ID */'L' + "0").getBytes())); - + // table tablet's prevrow Key tablePrevRowKey = new Key(tableExtent, TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnFamily(), TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnQualifier(), 0); mfw.append(tablePrevRowKey, KeyExtent.encodePrevEndRow(null)); - + // ----------] default tablet info Text defaultExtent = new Text(KeyExtent.getMetadataEntry(new Text(MetadataTable.ID), null)); - + // default's directory Key defaultDirKey = new Key(defaultExtent, TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnFamily(), TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnQualifier(), 0); mfw.append(defaultDirKey, new Value(Constants.DEFAULT_TABLET_LOCATION.getBytes())); - + // default's time Key defaultTimeKey = new Key(defaultExtent, TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnFamily(), TabletsSection.ServerColumnFamily.TIME_COLUMN.getColumnQualifier(), 0); mfw.append(defaultTimeKey, new Value((/* TabletTime.LOGICAL_TIME_ID */'L' + "0").getBytes())); - + // default's prevrow Key defaultPrevRowKey = new Key(defaultExtent, TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnFamily(), TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.getColumnQualifier(), 0); mfw.append(defaultPrevRowKey, KeyExtent.encodePrevEndRow(MetadataSchema.TabletsSection.getRange().getEndKey().getRow())); - + testRfile.closeWriter(); - + if (true) { - FileOutputStream fileOutputStream = new FileOutputStream(new File("/tmp/testEncryptedRootFile.rf")); + FileOutputStream fileOutputStream = new FileOutputStream(tempFolder.newFile("testEncryptedRootFile.rf")); fileOutputStream.write(testRfile.baos.toByteArray()); fileOutputStream.flush(); fileOutputStream.close(); } - + testRfile.openReader(); testRfile.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false); assertTrue(testRfile.iter.hasTop()); - + assertTrue(testRfile.reader.getLastKey() != null); - + testRfile.closeReader(); - + if (oldSiteConfigProperty != null) { System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, oldSiteConfigProperty); } else { System.clearProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP); } ((SiteConfiguration) conf).clearAndNull(); - + } }