accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vi...@apache.org
Subject git commit: ACCUMULO-1598 Fixes RFlile unit tests
Date Thu, 31 Oct 2013 17:40:14 GMT
Updated Branches:
  refs/heads/master b0da55a65 -> 48fae074d


ACCUMULO-1598 Fixes RFlile unit tests

The RFile tests had several tests in the @Ignore state.  Many
of these were redundant, so I simply excise them.  There were
a couple of missing files from the repository that the tests
needed; this patch adds them.

Signed-off-by: John Vines <jvines@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/48fae074
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/48fae074
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/48fae074

Branch: refs/heads/master
Commit: 48fae074de4f029966cbbcb90b1da8dd9ad8016b
Parents: b0da55a
Author: Michael Allen <michael@sqrrl.com>
Authored: Thu Oct 31 13:17:58 2013 -0400
Committer: John Vines <jvines@gmail.com>
Committed: Thu Oct 31 13:40:09 2013 -0400

----------------------------------------------------------------------
 .../accumulo/core/file/rfile/RFileTest.java     | 268 -------------------
 .../test/resources/crypto-on-accumulo-site.xml  | 164 ++++++++++++
 ...rypto-on-no-key-encryption-accumulo-site.xml | 144 ++++++++++
 3 files changed, 308 insertions(+), 268 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/48fae074/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index e049531..afc94c6 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@ -68,7 +68,6 @@ import org.apache.hadoop.fs.Seekable;
 import org.apache.hadoop.io.Text;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -1805,7 +1804,6 @@ public class RFileTest {
   }
 
   @Test
-  @Ignore
   public void testEncryptedRFiles() throws Exception {
     String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
     @SuppressWarnings("deprecation")
@@ -1831,272 +1829,6 @@ public class RFileTest {
   }
 
   @Test
-  @Ignore
-  public void testRootTabletFromServer() throws Exception {
-    String oldSiteConfigProperty = System.getProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
-    @SuppressWarnings("deprecation")
-    AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration();
-    System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_ON_CONF);
-    ((SiteConfiguration) conf).clearAndNull();
-
-    TestRFile trf = new TestRFile();
-    trf.preGeneratedInputFile = tempFolder.newFile("should_work.rf");
-
-    trf.openReader();
-    trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
-    assertTrue(trf.iter.hasTop());
-
-    assertTrue(trf.reader.getLastKey() != null);
-
-    trf.closeReader();
-
-    if (oldSiteConfigProperty != null) {
-      System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, oldSiteConfigProperty);
-    } else {
-      System.clearProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP);
-    }
-    ((SiteConfiguration) conf).clearAndNull();
-
-  }
-
-  private static final int TOTAL_NUM_ROWS = 10;
-  private static final int ROW_MOD_VALUE = 10;
-
-  @Test
-  @Ignore
-  // These tests will purge the disk cache when the run, so it's not recommended that they
be run in development systems.
-  public void testEncryptedRFileWriteSpeed() throws Exception {
-
-    System.out
-        .println("Unencrypted Write, Unencrypted Read (Cache), Unencrypted Read (FS only),
Encrypted Write, Encrypted Read (Cache), Encrypted Read (FS Only)");
-    int numIterations = 1;
-
-    for (int i = 0; i < numIterations; i++) {
-      @SuppressWarnings("deprecation")
-      AccumuloConfiguration conf = AccumuloConfiguration.getSiteConfiguration();
-      System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_OFF_CONF);
-      ((SiteConfiguration) conf).clearAndNull();
-
-      TestRFile trf = new TestRFile();
-      trf.outputFile = tempFolder.newFile("testUnencryptedRfile.rf");
-      trf.openWriter();
-
-      double timeTickSize = 1000.0;
-      int numRowsRead = 0;
-
-      try {
-
-        performUnencryptedTests(trf, TOTAL_NUM_ROWS, ROW_MOD_VALUE, timeTickSize, true);
-
-        performEncryptedTests(TOTAL_NUM_ROWS, ROW_MOD_VALUE, timeTickSize, numRowsRead, false);
-
-      } catch (Exception e) {
-        e.printStackTrace();
-        throw e;
-      }
-
-    }
-
-  }
-
-  private void performUnencryptedTests(TestRFile trf, int totalNumRows, int rowModValue,
double timeTickSize, boolean first) throws IOException,
-      InterruptedException {
-    long start = System.currentTimeMillis();
-
-    writeRowsToRfile(trf, totalNumRows, rowModValue);
-
-    long end = System.currentTimeMillis();
-
-    System.out.print("" + ((end - start) / timeTickSize) + ", ");
-
-    trf.preGeneratedInputFile = trf.outputFile;
-
-    start = System.currentTimeMillis();
-
-    trf.openReader();
-    trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
-    int numRowsRead = 0;
-
-    int numRowsToRead = totalNumRows;
-    while (numRowsRead < TOTAL_NUM_ROWS) {
-      int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead);
-
-      numRowsToRead -= numRowsReadThisTime;
-      numRowsRead += numRowsReadThisTime;
-    }
-
-    trf.closeReader();
-
-    end = System.currentTimeMillis();
-
-    System.out.print("" + ((end - start) / timeTickSize) + ", ");
-
-    Runtime.getRuntime().exec("purge").waitFor();
-
-    start = System.currentTimeMillis();
-
-    trf.openReader();
-    trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
-    numRowsRead = 0;
-
-    numRowsToRead = totalNumRows;
-    while (numRowsRead < TOTAL_NUM_ROWS) {
-      int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead);
-
-      numRowsToRead -= numRowsReadThisTime;
-      numRowsRead += numRowsReadThisTime;
-    }
-
-    trf.closeReader();
-
-    end = System.currentTimeMillis();
-
-    if (first) {
-      System.out.print("" + ((end - start) / timeTickSize) + ", ");
-    } else {
-      System.out.println("" + ((end - start) / timeTickSize));
-
-    }
-
-  }
-
-  @SuppressWarnings("deprecation")
-  private void performEncryptedTests(int totalNumRows, int rowModValue, double timeTickSize,
int numRowsRead, boolean first) throws IOException,
-      InterruptedException {
-    AccumuloConfiguration conf;
-    TestRFile trf;
-    long start;
-    long end;
-    int numRowsToRead;
-
-    conf = AccumuloConfiguration.getSiteConfiguration();
-    System.setProperty(CryptoTest.CONFIG_FILE_SYSTEM_PROP, CryptoTest.CRYPTO_ON_CONF);
-    ((SiteConfiguration) conf).clearAndNull();
-
-    trf = new TestRFile();
-    trf.outputFile = tempFolder.newFile("testEncryptedRfile.rf");
-    trf.openWriter();
-
-    start = System.currentTimeMillis();
-
-    writeRowsToRfile(trf, totalNumRows, rowModValue);
-
-    end = System.currentTimeMillis();
-
-    System.out.print("" + ((end - start) / timeTickSize) + ", ");
-
-    trf.preGeneratedInputFile = trf.outputFile;
-
-    start = System.currentTimeMillis();
-
-    trf.openReader();
-    trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
-
-    numRowsToRead = totalNumRows;
-    while (numRowsRead < TOTAL_NUM_ROWS) {
-      int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead);
-
-      numRowsToRead -= numRowsReadThisTime;
-      numRowsRead += numRowsReadThisTime;
-    }
-
-    trf.closeReader();
-
-    end = System.currentTimeMillis();
-
-    System.out.print("" + ((end - start) / timeTickSize) + ", ");
-
-    Runtime.getRuntime().exec("purge").waitFor();
-
-    start = System.currentTimeMillis();
-
-    trf.openReader();
-    trf.iter.seek(new Range((Key) null, null), EMPTY_COL_FAMS, false);
-    numRowsRead = 0;
-
-    numRowsToRead = totalNumRows;
-    while (numRowsRead < TOTAL_NUM_ROWS) {
-      int numRowsReadThisTime = readRandomRowsFromRfile(trf, totalNumRows, numRowsToRead);
-
-      numRowsToRead -= numRowsReadThisTime;
-      numRowsRead += numRowsReadThisTime;
-    }
-
-    trf.closeReader();
-
-    end = System.currentTimeMillis();
-
-    if (first) {
-      System.out.print("" + ((end - start) / timeTickSize) + ", ");
-    } else {
-      System.out.println("" + ((end - start) / timeTickSize));
-
-    }
-
-  }
-
-  private int readRandomRowsFromRfile(TestRFile trf, int totalRowCount, int maxRowsToRead)
throws IOException {
-    if (maxRowsToRead <= 0) {
-      return 0;
-    }
-
-    int numRowsRead = 0;
-    Random rand = new Random(System.nanoTime());
-
-    int firstKeyNum = Math.abs(rand.nextInt()) % totalRowCount;
-    // int lastKeyNum = Math.abs(rand.nextInt()) % totalRowCount;
-    int lastKeyNum = firstKeyNum + 1;
-
-    if (lastKeyNum >= totalRowCount) {
-      lastKeyNum = firstKeyNum;
-    }
-
-    if (lastKeyNum < firstKeyNum) {
-      int temp = lastKeyNum;
-      lastKeyNum = firstKeyNum;
-      firstKeyNum = temp;
-    }
-
-    if (lastKeyNum - firstKeyNum > maxRowsToRead) {
-      lastKeyNum = firstKeyNum + maxRowsToRead;
-    }
-
-    Key firstKey = nk(nf("r_", firstKeyNum), "cf_0", "cq_0", "vis", 0L);
-    Key lastKey = nk(nf("r_", lastKeyNum), "cf_19", "cq_19", "vis", 0L);
-
-    trf.iter.seek(new Range(firstKey, lastKey), EMPTY_COL_FAMS, false);
-    for (int i = firstKeyNum; i < lastKeyNum; i++) {
-      @SuppressWarnings("unused")
-      Key k = trf.iter.getTopKey();
-      @SuppressWarnings("unused")
-      Value v = trf.iter.getTopValue();
-
-      trf.iter.next();
-
-      numRowsRead++;
-    }
-
-    return numRowsRead;
-
-  }
-
-  private void writeRowsToRfile(TestRFile trf, int numRowsToWriteAndRead, int rowModValue)
throws IOException {
-    for (int i = 0; i < numRowsToWriteAndRead; i++) {
-      String rowID = nf("r_", (i % rowModValue));
-      String colFam = nf("cf_", (i % 20));
-      String colQual = nf("cq_", (i % 20));
-      String colVis = "vis";
-
-      Key k = nk(rowID, colFam, colQual, colVis, i);
-      Value v = nv("" + i);
-
-      trf.writer.append(k, v);
-    }
-
-    trf.closeWriter();
-  }
-
-  @Test
   public void testRootTabletEncryption() throws Exception {
 
     // This tests that the normal set of operations used to populate a root tablet

http://git-wip-us.apache.org/repos/asf/accumulo/blob/48fae074/core/src/test/resources/crypto-on-accumulo-site.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/crypto-on-accumulo-site.xml b/core/src/test/resources/crypto-on-accumulo-site.xml
new file mode 100644
index 0000000..9dc4aac
--- /dev/null
+++ b/core/src/test/resources/crypto-on-accumulo-site.xml
@@ -0,0 +1,164 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+	<!--
+  Put your site-specific accumulo configurations here.
+
+  The available configuration values along with their defaults
+  are documented in docs/config.html
+
+  Unless you are simply testing at your workstation, you will most 
+  definitely need to change the three entries below.
+	-->
+
+    <property>
+      <name>instance.zookeeper.host</name>
+      <value>localhost:2181</value>
+      <description>comma separated list of zookeeper servers</description>
+    </property>
+
+    <property>
+      <name>logger.dir.walog</name>
+      <value>walogs</value>
+      <description>The directory used to store write-ahead logs on the local filesystem.
It is possible to specify a comma-separated list of directories.</description>
+    </property>
+    
+    <property>
+      <name>instance.secret</name>
+      <value>DEFAULT</value>
+      <description>A secret unique to a given instance that all servers must know in
order to communicate with one another. 
+                   Change it before initialization. To change it later use ./bin/accumulo
org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd], 
+                   and then update this file.
+      </description>
+    </property>
+
+    <property>
+      <name>tserver.memory.maps.max</name>
+      <value>80M</value>
+    </property>
+    
+    <property>
+      <name>tserver.cache.data.size</name>
+      <value>7M</value>
+    </property>
+    
+    <property>
+      <name>tserver.cache.index.size</name>
+      <value>20M</value>
+    </property>
+    
+    <property>
+      <name>trace.password</name>
+      <!-- 
+        change this to the root user's password, and/or change the user below 
+       -->
+      <value>password</value>
+    </property>
+    
+    <property>
+      <name>trace.user</name>
+      <value>root</value>
+    </property>
+    
+    <property>
+      <name>tserver.sort.buffer.size</name>
+      <value>50M</value>
+    </property>
+    
+    <property>
+      <name>tserver.walog.max.size</name>
+      <value>100M</value>
+    </property>
+
+    <property>
+      <name>general.classpaths</name>
+      <value>
+    $ACCUMULO_HOME/server/target/classes/,
+    $ACCUMULO_HOME/core/target/classes/,
+    $ACCUMULO_HOME/start/target/classes/,
+    $ACCUMULO_HOME/fate/target/classes/,
+    $ACCUMULO_HOME/proxy/target/classes/,
+    $ACCUMULO_HOME/examples/target/classes/,
+	$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
+	$ACCUMULO_HOME/lib/[^.].*.jar,
+	$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+	$HADOOP_CONF_DIR,
+	$HADOOP_PREFIX/[^.].*.jar,
+	$HADOOP_PREFIX/lib/[^.].*.jar,
+      </value>
+      <description>Classpaths that accumulo checks for updates and class files.
+      When using the Security Manager, please remove the ".../target/classes/" values.
+      </description>
+    </property>
+
+    <property>
+      <name>crypto.module.class</name>
+      <value>org.apache.accumulo.core.security.crypto.DefaultCryptoModule</value>
+    </property>
+    <property>
+      <name>crypto.cipher.suite</name>
+      <value>AES/CFB/PKCS5Padding</value>
+    </property>
+    <property>
+      <name>crypto.cipher.algorithm.name</name>
+      <value>AES</value>
+    </property>
+    <property>
+      <name>crypto.cipher.key.length</name>
+      <value>128</value>
+    </property>
+    <property>
+      <name>crypto.secure.rng</name>
+      <value>SHA1PRNG</value>
+    </property>
+    <property>
+      <name>crypto.secure.rng.provider</name>
+      <value>SUN</value>
+    </property>
+    <property>
+      <name>crypto.secret.key.encryption.strategy.class</name>
+      <value>org.apache.accumulo.core.security.crypto.DefaultSecretKeyEncryptionStrategy</value>
+    </property>
+    <property>
+      <name>instance.dfs.dir</name>
+      <value>/tmp</value>
+    </property>
+    <property>
+      <name>instance.dfs.uri</name>
+      <value>file:///</value>
+    </property>
+    
+    <property>
+      <name>crypto.default.key.strategy.hdfs.uri</name>
+      <value>file:///</value>
+    </property>
+    <property>
+      <name>crypto.default.key.strategy.key.location</name>
+      <value>/tmp/test.secret.key</value>
+    </property>
+    
+    <property>
+    	<name>crypto.default.key.strategy.cipher.suite</name>
+    	<value>AES/ECB/NoPadding</value>
+    </property>
+
+
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/accumulo/blob/48fae074/core/src/test/resources/crypto-on-no-key-encryption-accumulo-site.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/crypto-on-no-key-encryption-accumulo-site.xml b/core/src/test/resources/crypto-on-no-key-encryption-accumulo-site.xml
new file mode 100644
index 0000000..640abac
--- /dev/null
+++ b/core/src/test/resources/crypto-on-no-key-encryption-accumulo-site.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+	<!--
+  Put your site-specific accumulo configurations here.
+
+  The available configuration values along with their defaults
+  are documented in docs/config.html
+
+  Unless you are simply testing at your workstation, you will most
+  definitely need to change the three entries below.
+	-->
+
+    <property>
+      <name>instance.zookeeper.host</name>
+      <value>localhost:2181</value>
+      <description>comma separated list of zookeeper servers</description>
+    </property>
+
+    <property>
+      <name>logger.dir.walog</name>
+      <value>walogs</value>
+      <description>The directory used to store write-ahead logs on the local filesystem.
It is possible to specify a comma-separated list of directories.</description>
+    </property>
+
+    <property>
+      <name>instance.secret</name>
+      <value>DEFAULT</value>
+      <description>A secret unique to a given instance that all servers must know in
order to communicate with one another.
+                   Change it before initialization. To change it later use ./bin/accumulo
org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd],
+                   and then update this file.
+      </description>
+    </property>
+
+    <property>
+      <name>tserver.memory.maps.max</name>
+      <value>80M</value>
+    </property>
+
+    <property>
+      <name>tserver.cache.data.size</name>
+      <value>7M</value>
+    </property>
+
+    <property>
+      <name>tserver.cache.index.size</name>
+      <value>20M</value>
+    </property>
+
+    <property>
+      <name>trace.password</name>
+      <!--
+        change this to the root user's password, and/or change the user below
+       -->
+      <value>password</value>
+    </property>
+
+    <property>
+      <name>trace.user</name>
+      <value>root</value>
+    </property>
+
+    <property>
+      <name>tserver.sort.buffer.size</name>
+      <value>50M</value>
+    </property>
+
+    <property>
+      <name>tserver.walog.max.size</name>
+      <value>100M</value>
+    </property>
+
+    <property>
+      <name>general.classpaths</name>
+      <value>
+    $ACCUMULO_HOME/server/target/classes/,
+    $ACCUMULO_HOME/core/target/classes/,
+    $ACCUMULO_HOME/start/target/classes/,
+    $ACCUMULO_HOME/fate/target/classes/,
+    $ACCUMULO_HOME/proxy/target/classes/,
+    $ACCUMULO_HOME/examples/target/classes/,
+	$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar,
+	$ACCUMULO_HOME/lib/[^.].*.jar,
+	$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
+	$HADOOP_CONF_DIR,
+	$HADOOP_PREFIX/[^.].*.jar,
+	$HADOOP_PREFIX/lib/[^.].*.jar,
+      </value>
+      <description>Classpaths that accumulo checks for updates and class files.
+      When using the Security Manager, please remove the ".../target/classes/" values.
+      </description>
+    </property>
+
+    <property>
+      <name>crypto.module.class</name>
+      <value>org.apache.accumulo.core.security.crypto.DefaultCryptoModule</value>
+    </property>
+    <property>
+      <name>crypto.cipher.suite</name>
+      <value>AES/CFB/PKCS5Padding</value>
+    </property>
+    <property>
+      <name>crypto.cipher.algorithm.name</name>
+      <value>AES</value>
+    </property>
+    <property>
+      <name>crypto.cipher.key.length</name>
+      <value>128</value>
+    </property>
+    <property>
+      <name>crypto.secure.rng</name>
+      <value>SHA1PRNG</value>
+    </property>
+    <property>
+      <name>crypto.secure.rng.provider</name>
+      <value>SUN</value>
+    </property>
+    <property>
+      <name>instance.dfs.dir</name>
+      <value>/tmp</value>
+    </property>
+    <property>
+      <name>instance.dfs.uri</name>
+      <value>file:///</value>
+    </property>
+
+</configuration>


Mime
View raw message