accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject git commit: ACCUMULO-1689 Add a simple test to make sure we get rid of some files after compacting tables which we wrote data to.
Date Tue, 10 Sep 2013 03:40:58 GMT
Updated Branches:
  refs/heads/1.4.5-SNAPSHOT f868c3762 -> 76c3c8490


ACCUMULO-1689 Add a simple test to make sure we get rid of some files after
compacting tables which we wrote data to.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/76c3c849
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/76c3c849
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/76c3c849

Branch: refs/heads/1.4.5-SNAPSHOT
Commit: 76c3c8490df6e5a81f40d5db7c084fd4d5d716b2
Parents: f868c37
Author: Josh Elser <elserj@apache.org>
Authored: Mon Sep 9 23:41:01 2013 -0400
Committer: Josh Elser <elserj@apache.org>
Committed: Mon Sep 9 23:41:01 2013 -0400

----------------------------------------------------------------------
 pom.xml                                         |   6 +
 src/minicluster/pom.xml                         |   5 +
 .../minicluster/MiniAccumuloClusterGCTest.java  | 119 +++++++++++++++++++
 3 files changed, 130 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/76c3c849/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d427da8..9ed2fdf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -651,6 +651,12 @@
         <version>4.11</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>com.google.guava</groupId>
+        <artifactId>guava</artifactId>
+        <version>14.0.1</version>
+        <scope>test</scope>
+      </dependency>
 
       <!-- provided dependencies needed at runtime -->
       <dependency>

http://git-wip-us.apache.org/repos/asf/accumulo/blob/76c3c849/src/minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/src/minicluster/pom.xml b/src/minicluster/pom.xml
index bb85148..3e23b0c 100644
--- a/src/minicluster/pom.xml
+++ b/src/minicluster/pom.xml
@@ -104,6 +104,11 @@
       <groupId>commons-lang</groupId>
       <artifactId>commons-lang</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
 </project>

http://git-wip-us.apache.org/repos/asf/accumulo/blob/76c3c849/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
----------------------------------------------------------------------
diff --git a/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
b/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
new file mode 100644
index 0000000..7c57e7e
--- /dev/null
+++ b/src/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterGCTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.minicluster;
+
+import java.io.File;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.SuffixFileFilter;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * 
+ */
+public class MiniAccumuloClusterGCTest {
+  
+  private static TemporaryFolder tmpDir = new TemporaryFolder();
+  private static MiniAccumuloConfig macConfig;
+  private static MiniAccumuloCluster accumulo;
+  private static final String passwd = "password";
+  
+  @BeforeClass
+  public static void setupMiniCluster() throws Exception {
+    tmpDir.create();
+    Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR);
+    
+    macConfig = new MiniAccumuloConfig(tmpDir.getRoot(), passwd);
+    macConfig.setNumTservers(1);
+    
+    // Turn on the garbage collector
+    macConfig.runGC(true);
+    
+    // And tweak the settings to make it run often
+    Map<String,String> config = ImmutableMap.of(Property.GC_CYCLE_DELAY.getKey(), "1s",
Property.GC_CYCLE_START.getKey(), "0s");
+    macConfig.setSiteConfig(config);
+    
+    accumulo = new MiniAccumuloCluster(macConfig);
+    accumulo.start();
+  }
+  
+  @AfterClass
+  public static void tearDownMiniCluster() throws Exception {
+    accumulo.stop();
+    tmpDir.delete();
+  }
+  
+  @Test(timeout = 20000)
+  public void test() throws Exception {
+    ZooKeeperInstance inst = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
+    Connector c = inst.getConnector("root", passwd);
+    
+    final String table = "foobar";
+    c.tableOperations().create(table);
+    
+    BatchWriter bw = null;
+    
+    // Add some data
+    try {
+      bw = c.createBatchWriter(table, 1000l, 100l, 1);
+      Mutation m = new Mutation("a");
+      for (int i = 0; i < 50; i++) {
+        m.put("colf", Integer.toString(i), "");
+      }
+      
+      bw.addMutation(m);
+    } finally {
+      if (null != bw) {
+        bw.close();
+      }
+    }
+    
+    final boolean flush = true, wait = true;
+    
+    // Compact the tables to get some rfiles which we can gc
+    c.tableOperations().compact(table, null, null, flush, wait);
+    c.tableOperations().compact("!METADATA", null, null, flush, wait);
+    
+    File accumuloDir = new File(tmpDir.getRoot().getAbsolutePath(), "accumulo");
+    File tables = new File(accumuloDir.getAbsolutePath(), "tables");
+    
+    int fileCountAfterCompaction = FileUtils.listFiles(tables, new SuffixFileFilter(".rf"),
TrueFileFilter.TRUE).size();
+    
+    // Sleep for 4s to let the GC do its thing
+    Thread.sleep(4000);
+    
+    int fileCountAfterGCWait = FileUtils.listFiles(tables, new SuffixFileFilter(".rf"), TrueFileFilter.TRUE).size();
+    
+    Assert.assertTrue("Expected to find less files after compaction and pause for GC", fileCountAfterGCWait
< fileCountAfterCompaction);
+  }
+  
+}


Mime
View raw message