accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [01/11] git commit: ACCUMULO-2163 fix GC IT test and start() only run gc if not already started
Date Mon, 13 Jan 2014 17:58:26 GMT
Updated Branches:
  refs/heads/1.4.5-SNAPSHOT 1431d3485 -> 8e0e0e1f5
  refs/heads/1.5.1-SNAPSHOT aa5c92189 -> 0603edbbe
  refs/heads/1.6.0-SNAPSHOT b54266a8a -> 9a81ca2d9
  refs/heads/master 4cffb74a5 -> 7345b8333


ACCUMULO-2163 fix GC IT test and start() only run gc if not already started


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b54266a8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b54266a8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b54266a8

Branch: refs/heads/master
Commit: b54266a8a4da7a14d77416f9d0acf3a937e4bb60
Parents: 0ab78c6
Author: Keith Turner <kturner@apache.org>
Authored: Fri Jan 10 20:28:08 2014 -0500
Committer: Keith Turner <kturner@apache.org>
Committed: Mon Jan 13 10:52:15 2014 -0500

----------------------------------------------------------------------
 .../minicluster/MiniAccumuloCluster.java        |   5 +-
 .../test/functional/GarbageCollectorIT.java     | 122 +++++++++++--------
 2 files changed, 74 insertions(+), 53 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/b54266a8/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index 09af10b..a2c34d4 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@ -480,7 +480,10 @@ public class MiniAccumuloCluster {
     if (masterProcess == null) {
       masterProcess = _exec(Master.class, ServerType.MASTER);
     }
-    gcProcess = _exec(SimpleGarbageCollector.class, ServerType.GARBAGE_COLLECTOR);
+
+    if (gcProcess == null) {
+      gcProcess = _exec(SimpleGarbageCollector.class, ServerType.GARBAGE_COLLECTOR);
+    }
   }
 
   private List<String> buildRemoteDebugParams(int port) {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/b54266a8/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index b813ca9..44c57c1 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@ -16,6 +16,7 @@
  */
 package org.apache.accumulo.test.functional;
 
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.util.Collections;
@@ -31,6 +32,7 @@ import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
@@ -44,9 +46,11 @@ import org.apache.accumulo.core.util.ServerServices;
 import org.apache.accumulo.core.util.ServerServices.Service;
 import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooLock;
 import org.apache.accumulo.gc.SimpleGarbageCollector;
 import org.apache.accumulo.minicluster.MemoryUnit;
 import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.minicluster.ProcessNotFoundException;
 import org.apache.accumulo.minicluster.ProcessReference;
 import org.apache.accumulo.minicluster.ServerType;
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
@@ -56,6 +60,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.junit.Assert;
 import org.junit.Test;
@@ -76,8 +81,24 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
     cfg.useMiniDFS();
   }
 
+  private void killMacGc() throws ProcessNotFoundException, InterruptedException, KeeperException
{
+    // kill gc started by MAC
+    getCluster().killProcess(ServerType.GARBAGE_COLLECTOR, getCluster().getProcesses().get(ServerType.GARBAGE_COLLECTOR).iterator().next());
+    // delete lock in zookeeper if there, this will allow next GC to start quickly
+    String path = ZooUtil.getRoot(new ZooKeeperInstance(getCluster().getClientConfig()))
+ Constants.ZGC_LOCK;
+    ZooReaderWriter zk = new ZooReaderWriter(cluster.getZooKeepers(), 30000, OUR_SECRET);
+    try {
+      ZooLock.deleteLock(zk, path);
+    } catch (IllegalStateException e) {
+
+    }
+
+    assertNull(getCluster().getProcesses().get(ServerType.GARBAGE_COLLECTOR));
+  }
+
   @Test(timeout = 4 * 60 * 1000)
   public void gcTest() throws Exception {
+    killMacGc();
     Connector c = getConnector();
     c.tableOperations().create("test_ingest");
     c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(),
"5K");
@@ -95,19 +116,19 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
         break;
       before = more;
     }
-    Process gc = cluster.exec(SimpleGarbageCollector.class);
-    try {
-      UtilWaitThread.sleep(10 * 1000);
-      int after = countFiles();
-      VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
-      assertTrue(after < before);
-    } finally {
-      gc.destroy();
-    }
+
+    // restart GC
+    getCluster().start();
+    UtilWaitThread.sleep(10 * 1000);
+    int after = countFiles();
+    VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+    assertTrue(after < before);
   }
 
   @Test(timeout = 4 * 60 * 1000)
   public void gcLotsOfCandidatesIT() throws Exception {
+    killMacGc();
+
     log.info("Filling metadata table with bogus delete flags");
     Connector c = getConnector();
     addEntries(c, new BatchWriterOpts());
@@ -121,14 +142,15 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
 
   @Test(timeout = 20 * 60 * 1000)
   public void dontGCRootLog() throws Exception {
+    killMacGc();
     // dirty metadata
     Connector c = getConnector();
     String table = getTableNames(1)[0];
     c.tableOperations().create(table);
     // let gc run for a bit
-    Process gc = cluster.exec(SimpleGarbageCollector.class);
+    cluster.start();
     UtilWaitThread.sleep(20 * 1000);
-    gc.destroy();
+    killMacGc();
     // kill tservers
     for (ProcessReference ref : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
       cluster.killProcess(ServerType.TABLET_SERVER, ref);
@@ -145,52 +167,48 @@ public class GarbageCollectorIT extends ConfigurableMacIT {
 
   @Test(timeout = 60 * 1000)
   public void testProperPortAdvertisement() throws Exception {
-    Process gc = cluster.exec(SimpleGarbageCollector.class);
+
     Connector conn = getConnector();
     Instance instance = conn.getInstance();
-    
-    try {
-      ZooReaderWriter zk = new ZooReaderWriter(cluster.getZooKeepers(), 30000, OUR_SECRET);
-      String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
-      for (int i = 0; i < 5; i++) {
-        List<String> locks;
-        try {
-          locks = zk.getChildren(path, null);
-        } catch (NoNodeException e ) {
-          Thread.sleep(5000);
-          continue;
-        }
-  
-        if (locks != null && locks.size() > 0) {
-          Collections.sort(locks);
-          
-          String lockPath = path + "/" + locks.get(0);
-          
-          String gcLoc = new String(zk.getData(lockPath, null));
-  
-          Assert.assertTrue("Found unexpected data in zookeeper for GC location: " + gcLoc,
gcLoc.startsWith(Service.GC_CLIENT.name()));
-          int loc = gcLoc.indexOf(ServerServices.SEPARATOR_CHAR);
-          Assert.assertNotEquals("Could not find split point of GC location for: " + gcLoc,
-1, loc);
-          String addr = gcLoc.substring(loc + 1);
-          
-          int addrSplit = addr.indexOf(':');
-          Assert.assertNotEquals("Could not find split of GC host:port for: " + addr, -1,
addrSplit);
-          
-          String host = addr.substring(0, addrSplit), port = addr.substring(addrSplit + 1);
-          // We shouldn't have the "bindall" address in zk
-          Assert.assertNotEquals("0.0.0.0", host);
-          // Nor should we have the "random port" in zk
-          Assert.assertNotEquals(0, Integer.parseInt(port));
-          return;
-        }
-        
+
+    ZooReaderWriter zk = new ZooReaderWriter(cluster.getZooKeepers(), 30000, OUR_SECRET);
+    String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
+    for (int i = 0; i < 5; i++) {
+      List<String> locks;
+      try {
+        locks = zk.getChildren(path, null);
+      } catch (NoNodeException e) {
         Thread.sleep(5000);
+        continue;
+      }
+
+      if (locks != null && locks.size() > 0) {
+        Collections.sort(locks);
+
+        String lockPath = path + "/" + locks.get(0);
+
+        String gcLoc = new String(zk.getData(lockPath, null));
+
+        Assert.assertTrue("Found unexpected data in zookeeper for GC location: " + gcLoc,
gcLoc.startsWith(Service.GC_CLIENT.name()));
+        int loc = gcLoc.indexOf(ServerServices.SEPARATOR_CHAR);
+        Assert.assertNotEquals("Could not find split point of GC location for: " + gcLoc,
-1, loc);
+        String addr = gcLoc.substring(loc + 1);
+
+        int addrSplit = addr.indexOf(':');
+        Assert.assertNotEquals("Could not find split of GC host:port for: " + addr, -1, addrSplit);
+
+        String host = addr.substring(0, addrSplit), port = addr.substring(addrSplit + 1);
+        // We shouldn't have the "bindall" address in zk
+        Assert.assertNotEquals("0.0.0.0", host);
+        // Nor should we have the "random port" in zk
+        Assert.assertNotEquals(0, Integer.parseInt(port));
+        return;
       }
-      
-      Assert.fail("Could not find advertised GC address");
-    } finally {
-      gc.destroy();
+
+      Thread.sleep(5000);
     }
+
+    Assert.fail("Could not find advertised GC address");
   }
 
   private int countFiles() throws Exception {


Mime
View raw message