hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject [1/2] hbase git commit: HBASE-16846 Procedure v2 - executor cleanup
Date Mon, 17 Oct 2016 17:48:26 GMT
Repository: hbase
Updated Branches:
  refs/heads/master c8e9a295c -> c6e9dabe6


http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestDelayedUtil.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestDelayedUtil.java
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestDelayedUtil.java
new file mode 100644
index 0000000..1e2db4d
--- /dev/null
+++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestDelayedUtil.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure2.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+
+@Category({MasterTests.class, SmallTests.class})
+public class TestDelayedUtil {
+  private static final Log LOG = LogFactory.getLog(TestDelayedUtil.class);
+
+  @Test
+  public void testDelayedContainerEquals() {
+    Object o1 = new Object();
+    Object o2 = new Object();
+    ZeroDelayContainer<Long> lnull = new ZeroDelayContainer(null);
+    ZeroDelayContainer<Long> l10a = new ZeroDelayContainer<Long>(10L);
+    ZeroDelayContainer<Long> l10b = new ZeroDelayContainer(10L);
+    ZeroDelayContainer<Long> l15 = new ZeroDelayContainer(15L);
+    ZeroDelayContainer<Object> onull = new ZeroDelayContainer<Object>(null);
+    ZeroDelayContainer<Object> o1ca = new ZeroDelayContainer<Object>(o1);
+    ZeroDelayContainer<Object> o1cb = new ZeroDelayContainer<Object>(o1);
+    ZeroDelayContainer<Object> o2c = new ZeroDelayContainer<Object>(o2);
+
+    ZeroDelayContainer[] items = new ZeroDelayContainer[] {
+      lnull, l10a, l10b, l15, onull, o1ca, o1cb, o2c,
+    };
+
+    assertContainersEquals(lnull, items, lnull, onull);
+    assertContainersEquals(l10a, items, l10a, l10b);
+    assertContainersEquals(l10b, items, l10a, l10b);
+    assertContainersEquals(l15, items, l15);
+    assertContainersEquals(onull, items, lnull, onull);
+    assertContainersEquals(o1ca, items, o1ca, o1cb);
+    assertContainersEquals(o1cb, items, o1ca, o1cb);
+    assertContainersEquals(o2c, items, o2c);
+  }
+
+  private void assertContainersEquals(final ZeroDelayContainer src,
+      final ZeroDelayContainer[] items, final ZeroDelayContainer... matches) {
+    for (int i = 0; i < items.length; ++i) {
+      boolean shouldMatch = false;
+      for (int j = 0; j < matches.length; ++j) {
+        if (items[i] == matches[j]) {
+          shouldMatch = true;
+          break;
+        }
+      }
+      boolean isMatching = src.equals(items[i]);
+      assertEquals(src.getObject() + " unexpectedly match " + items[i].getObject(),
+        shouldMatch, isMatching);
+    }
+  }
+
+  private static class ZeroDelayContainer<T> extends DelayedUtil.DelayedContainer<T>
{
+    public ZeroDelayContainer(final T object) {
+      super(object);
+    }
+
+    @Override
+    public long getTimeoutTimestamp() {
+      return 0;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index af5d03d..a8329e3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1035,6 +1035,7 @@ public class HMaster extends HRegionServer implements MasterServices
{
     procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this));
     procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore,
         procEnv.getProcedureQueue());
+    configurationManager.registerObserver(procEnv);
 
     final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS,
         Math.max(Runtime.getRuntime().availableProcessors(),
@@ -1048,6 +1049,7 @@ public class HMaster extends HRegionServer implements MasterServices
{
 
   private void stopProcedureExecutor() {
     if (procedureExecutor != null) {
+      configurationManager.deregisterObserver(procedureExecutor.getEnvironment());
       procedureExecutor.stop();
     }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
index 183b41d..9362f24 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.ipc.RpcServer;
@@ -42,7 +43,7 @@ import org.apache.hadoop.hbase.util.FSUtils;
 
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class MasterProcedureEnv {
+public class MasterProcedureEnv implements ConfigurationObserver {
   private static final Log LOG = LogFactory.getLog(MasterProcedureEnv.class);
 
   @InterfaceAudience.Private
@@ -161,4 +162,9 @@ public class MasterProcedureEnv {
       procSched.suspendEvent(event);
     }
   }
+
+  @Override
+  public void onConfigurationChange(Configuration conf) {
+    master.getMasterProcedureExecutor().refreshConfiguration(conf);
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index cc9efd2..a7e1fb3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -191,6 +191,10 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
       serverBuckets[i] = null;
     }
 
+    // Remove Namespaces
+    clear(namespaceMap, null, NAMESPACE_QUEUE_KEY_COMPARATOR);
+    namespaceMap = null;
+
     // Remove Tables
     clear(tableMap, tableRunQueue, TABLE_QUEUE_KEY_COMPARATOR);
     tableMap = null;
@@ -204,12 +208,12 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler
{
       Queue<T> node = AvlTree.getFirst(treeMap);
       assert !node.isSuspended() : "can't clear suspended " + node.getKey();
       treeMap = AvlTree.remove(treeMap, node.getKey(), comparator);
-      removeFromRunQueue(fairq, node);
+      if (fairq != null) removeFromRunQueue(fairq, node);
     }
   }
 
   @Override
-  public int queueSize() {
+  protected int queueSize() {
     int count = 0;
 
     // Server queues

http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java
index 6e4f3cd..ee9b8fc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureSchedulerPerformanceEvaluation.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.master.TableLockManager;
 import org.apache.hadoop.hbase.procedure2.Procedure;
 import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.TestProcedure;
+import org.apache.hadoop.hbase.procedure2.util.StringUtils;
 import org.apache.hadoop.hbase.util.AbstractHBaseTool;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -244,24 +245,29 @@ public class MasterProcedureSchedulerPerformanceEvaluation extends AbstractHBase
   protected int doWork() throws Exception {
     procedureScheduler = new MasterProcedureScheduler(
         UTIL.getConfiguration(), new TableLockManager.NullTableLockManager());
+    procedureScheduler.start();
     setupOperations();
 
     final Thread[] threads = new Thread[numThreads];
     for (int i = 0; i < numThreads; ++i) {
       threads[i] = new AddProcsWorker();
     }
-    final float addBackTime = runThreads(threads) / 1000.0f;
+    final long addBackTime = runThreads(threads);
     System.out.println("Added " + numOps + " procedures to scheduler.");
 
     for (int i = 0; i < numThreads; ++i) {
       threads[i] = new PollAndLockWorker();
     }
-    final float pollTime = runThreads(threads) / 1000.0f;
+    final long pollTime = runThreads(threads);
+    procedureScheduler.stop();
+
+    final float pollTimeSec = pollTime / 1000.0f;
+    final float addBackTimeSec = addBackTime / 1000.0f;
     System.out.println("******************************************");
-    System.out.println("Time - addBack     : " + addBackTime + "sec");
-    System.out.println("Ops/sec - addBack  : " + ((float)numOps / addBackTime));
-    System.out.println("Time - poll        : " + pollTime + "sec");
-    System.out.println("Ops/sec - poll     : " + ((float)numOps / pollTime));
+    System.out.println("Time - addBack     : " + StringUtils.humanTimeDiff(addBackTime));
+    System.out.println("Ops/sec - addBack  : " + StringUtils.humanSize(numOps / addBackTimeSec));
+    System.out.println("Time - poll        : " + StringUtils.humanTimeDiff(pollTime));
+    System.out.println("Ops/sec - poll     : " + StringUtils.humanSize(numOps / pollTimeSec));
     System.out.println("Num Operations     : " + numOps);
     System.out.println();
     System.out.println("Completed          : " + completed.get());

http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
index bcacb48..73a87d6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
@@ -59,11 +59,13 @@ public class TestMasterProcedureScheduler {
   public void setUp() throws IOException {
     conf = HBaseConfiguration.create();
     queue = new MasterProcedureScheduler(conf, new TableLockManager.NullTableLockManager());
+    queue.start();
   }
 
   @After
   public void tearDown() throws IOException {
     assertEquals("proc-queue expected to be empty", 0, queue.size());
+    queue.stop();
     queue.clear();
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c6e9dabe/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java
index d6ddd13..d66ca85 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.master.TableLockManager;
 import org.apache.hadoop.hbase.master.procedure.TestMasterProcedureScheduler.TestTableProcedure;
 import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.util.StringUtils;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
 
@@ -55,11 +56,13 @@ public class TestMasterProcedureSchedulerConcurrency {
   public void setUp() throws IOException {
     conf = HBaseConfiguration.create();
     queue = new MasterProcedureScheduler(conf, new TableLockManager.NullTableLockManager());
+    queue.start();
   }
 
   @After
   public void tearDown() throws IOException {
     assertEquals("proc-queue expected to be empty", 0, queue.size());
+    queue.stop();
     queue.clear();
   }
 
@@ -207,6 +210,14 @@ public class TestMasterProcedureSchedulerConcurrency {
     }
   }
 
+  @Test(timeout=60000)
+  public void testMasterProcedureSchedulerPerformanceEvaluation() throws Exception {
+    // Make sure the tool does not get stuck
+    MasterProcedureSchedulerPerformanceEvaluation.main(new String[] {
+      "-num_ops", "1000"
+    });
+  }
+
   public static class TestTableProcSet {
     private final MasterProcedureScheduler queue;
 


Mime
View raw message