hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bus...@apache.org
Subject [4/6] hbase git commit: HBASE-17280 Add mechanism to control hbase cleaner behavior
Date Thu, 02 Feb 2017 04:24:31 GMT
http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b1f57b/hbase-protocol-shaded/src/main/protobuf/Master.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto
index e62f52c..e22695b 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -356,6 +356,28 @@ message IsCatalogJanitorEnabledResponse {
   required bool value = 1;
 }
 
+message RunCleanerChoreRequest {
+}
+
+message RunCleanerChoreResponse {
+  required bool cleaner_chore_ran = 1;
+}
+
+message SetCleanerChoreRunningRequest {
+  required bool on = 1;
+}
+
+message SetCleanerChoreRunningResponse {
+  optional bool prev_value = 1;
+}
+
+message IsCleanerChoreEnabledRequest {
+}
+
+message IsCleanerChoreEnabledResponse {
+  required bool value = 1;
+}
+
 message SnapshotRequest {
   required SnapshotDescription snapshot = 1;
 }
@@ -742,6 +764,22 @@ service MasterService {
   rpc IsCatalogJanitorEnabled(IsCatalogJanitorEnabledRequest)
      returns(IsCatalogJanitorEnabledResponse);
 
+  /** Get a run of the CleanerChore */
+  rpc RunCleanerChore(RunCleanerChoreRequest)
+     returns(RunCleanerChoreResponse);
+
+  /**
+   * Enable the CleanerChore on or off.
+   */
+  rpc SetCleanerChoreRunning(SetCleanerChoreRunningRequest)
+     returns(SetCleanerChoreRunningResponse);
+
+  /**
+   * Query whether the CleanerChore is enabled.
+   */
+  rpc IsCleanerChoreEnabled(IsCleanerChoreEnabledRequest)
+     returns(IsCleanerChoreEnabledResponse);
+
   /**
    * Call a master coprocessor endpoint
    */

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b1f57b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 913ec60..3374405 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -97,9 +97,9 @@ import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
 import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
 import org.apache.hadoop.hbase.master.cleaner.ReplicationMetaCleaner;
-import org.apache.hadoop.hbase.master.locking.LockManager;
 import org.apache.hadoop.hbase.master.cleaner.ReplicationZKNodeCleaner;
 import org.apache.hadoop.hbase.master.cleaner.ReplicationZKNodeCleanerChore;
+import org.apache.hadoop.hbase.master.locking.LockManager;
 import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan;
 import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType;
 import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer;
@@ -113,9 +113,9 @@ import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
 import org.apache.hadoop.hbase.master.procedure.EnableTableProcedure;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
 import org.apache.hadoop.hbase.master.procedure.MergeTableRegionsProcedure;
 import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure;
-import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
 import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
 import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
 import org.apache.hadoop.hbase.master.procedure.SplitTableRegionProcedure;
@@ -954,6 +954,20 @@ public class HMaster extends HRegionServer implements MasterServices
{
       catalogJanitorChore.getEnabled() : false;
   }
 
+  boolean isCleanerChoreEnabled() {
+    boolean hfileCleanerFlag = true, logCleanerFlag = true;
+
+    if (hfileCleaner != null) {
+      hfileCleanerFlag = hfileCleaner.getEnabled();
+    }
+
+    if (logCleaner != null) {
+      logCleanerFlag = logCleaner.getEnabled();
+    }
+
+    return (hfileCleanerFlag && logCleanerFlag);
+  }
+
   @Override
   public TableDescriptors getTableDescriptors() {
     return this.tableDescriptors;
@@ -2660,6 +2674,10 @@ public class HMaster extends HRegionServer implements MasterServices
{
     return this.hfileCleaner;
   }
 
+  public LogCleaner getLogCleaner() {
+    return this.logCleaner;
+  }
+
   /**
    * @return the underlying snapshot manager
    */

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b1f57b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 60b8b65..81d4169 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -521,6 +521,22 @@ public class MasterRpcServices extends RSRpcServices
   }
 
   @Override
+  public SetCleanerChoreRunningResponse setCleanerChoreRunning(RpcController c,
+                                                               SetCleanerChoreRunningRequest
req)
+    throws ServiceException {
+    try {
+      master.checkInitialized();
+    } catch (IOException ioe) {
+      throw new ServiceException(ioe);
+    }
+    boolean prevValue =
+      master.getLogCleaner().getEnabled() && master.getHFileCleaner().getEnabled();
+    master.getLogCleaner().setEnabled(req.getOn());
+    master.getHFileCleaner().setEnabled(req.getOn());
+    return SetCleanerChoreRunningResponse.newBuilder().setPrevValue(prevValue).build();
+  }
+
+  @Override
   public EnableTableResponse enableTable(RpcController controller,
       EnableTableRequest request) throws ServiceException {
     try {
@@ -873,6 +889,14 @@ public class MasterRpcServices extends RSRpcServices
   }
 
   @Override
+  public IsCleanerChoreEnabledResponse isCleanerChoreEnabled(RpcController c,
+                                                             IsCleanerChoreEnabledRequest
req)
+    throws ServiceException {
+    return IsCleanerChoreEnabledResponse.newBuilder().setValue(master.isCleanerChoreEnabled())
+                                        .build();
+  }
+
+  @Override
   public IsMasterRunningResponse isMasterRunning(RpcController c,
       IsMasterRunningRequest req) throws ServiceException {
     try {
@@ -1203,6 +1227,18 @@ public class MasterRpcServices extends RSRpcServices
   }
 
   @Override
+  public RunCleanerChoreResponse runCleanerChore(RpcController c, RunCleanerChoreRequest
req)
+    throws ServiceException {
+    try {
+      master.checkInitialized();
+      Boolean result = master.getHFileCleaner().runCleaner() && master.getLogCleaner().runCleaner();
+      return ResponseConverter.buildRunCleanerChoreResponse(result);
+    } catch (IOException ioe) {
+      throw new ServiceException(ioe);
+    }
+  }
+
+  @Override
   public SetBalancerRunningResponse setBalancerRunning(RpcController c,
       SetBalancerRunningRequest req) throws ServiceException {
     try {

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b1f57b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
index b094507..c6b6f62 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
@@ -36,6 +36,7 @@ import java.io.IOException;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * Abstract Cleaner that uses a chain of delegates to clean a directory of files
@@ -50,6 +51,7 @@ public abstract class CleanerChore<T extends FileCleanerDelegate>
extends Schedu
   private final Configuration conf;
   protected List<T> cleanersChain;
   protected Map<String, Object> params;
+  private AtomicBoolean enabled = new AtomicBoolean(true);
 
   public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration
conf,
                       FileSystem fs, Path oldFileDir, String confKey) {
@@ -128,6 +130,14 @@ public abstract class CleanerChore<T extends FileCleanerDelegate>
extends Schedu
 
   @Override
   protected void chore() {
+    if (getEnabled()) {
+      runCleaner();
+    } else {
+      LOG.debug("Cleaner chore disabled! Not cleaning.");
+    }
+  }
+
+  public Boolean runCleaner() {
     try {
       FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir);
       checkAndDeleteEntries(files);
@@ -135,7 +145,9 @@ public abstract class CleanerChore<T extends FileCleanerDelegate>
extends Schedu
       e = e instanceof RemoteException ?
               ((RemoteException)e).unwrapRemoteException() : e;
       LOG.warn("Error while cleaning the logs", e);
+      return false;
     }
+    return true;
   }
 
   /**
@@ -291,4 +303,15 @@ public abstract class CleanerChore<T extends FileCleanerDelegate>
extends Schedu
       }
     }
   }
+
+  /**
+   * @param enabled
+   */
+  public boolean setEnabled(final boolean enabled) {
+    return this.enabled.getAndSet(enabled);
+  }
+
+  public boolean getEnabled() {
+    return this.enabled.get();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b1f57b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
index 92c7bb6..5c76643 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestCleanerChore.java
@@ -289,6 +289,72 @@ public class TestCleanerChore {
     Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(FileStatus.class));
   }
 
+  @Test
+  public void testDeleteFileWithCleanerEnabled() throws Exception {
+    Stoppable stop = new StoppableImplementation();
+    Configuration conf = UTIL.getConfiguration();
+    Path testDir = UTIL.getDataTestDir();
+    FileSystem fs = UTIL.getTestFileSystem();
+    String confKey = "hbase.test.cleaner.delegates";
+    conf.set(confKey, AlwaysDelete.class.getName());
+
+    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir,
confKey);
+
+    // Enable cleaner
+    chore.setEnabled(true);
+
+    // create the directory layout in the directory to clean
+    Path parent = new Path(testDir, "parent");
+    Path child = new Path(parent, "child");
+    Path file = new Path(child, "someFile");
+    fs.mkdirs(child);
+
+    // touch a new file
+    fs.create(file).close();
+    assertTrue("Test file didn't get created.", fs.exists(file));
+
+    // run the chore
+    chore.chore();
+
+    // verify all the files got deleted
+    assertFalse("File didn't get deleted", fs.exists(file));
+    assertFalse("Empty directory didn't get deleted", fs.exists(child));
+    assertFalse("Empty directory didn't get deleted", fs.exists(parent));
+  }
+
+  @Test
+  public void testDeleteFileWithCleanerDisabled() throws Exception {
+    Stoppable stop = new StoppableImplementation();
+    Configuration conf = UTIL.getConfiguration();
+    Path testDir = UTIL.getDataTestDir();
+    FileSystem fs = UTIL.getTestFileSystem();
+    String confKey = "hbase.test.cleaner.delegates";
+    conf.set(confKey, AlwaysDelete.class.getName());
+
+    AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir,
confKey);
+
+    // Disable cleaner
+    chore.setEnabled(false);
+
+    // create the directory layout in the directory to clean
+    Path parent = new Path(testDir, "parent");
+    Path child = new Path(parent, "child");
+    Path file = new Path(child, "someFile");
+    fs.mkdirs(child);
+
+    // touch a new file
+    fs.create(file).close();
+    assertTrue("Test file didn't get created.", fs.exists(file));
+
+    // run the chore
+    chore.chore();
+
+    // verify all the files exist
+    assertTrue("File got deleted with cleaner disabled", fs.exists(file));
+    assertTrue("Directory got deleted", fs.exists(child));
+    assertTrue("Directory got deleted", fs.exists(parent));
+  }
+
   private static class AllValidPaths extends CleanerChore<BaseHFileCleanerDelegate>
{
 
     public AllValidPaths(String name, Stoppable s, Configuration conf, FileSystem fs,

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b1f57b/hbase-shell/src/main/ruby/hbase/admin.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
index 0718627..94b9d3e 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -243,6 +243,26 @@ module Hbase
     end
 
     #----------------------------------------------------------------------------------------------
+    # Request cleaner chore to run (for garbage collection of HFiles and WAL files)
+    def cleaner_chore_run()
+      @admin.runCleanerChore()
+    end
+
+    #----------------------------------------------------------------------------------------------
+    # Enable/disable the cleaner chore
+    # Returns previous cleaner switch setting.
+    def cleaner_chore_switch(enableDisable)
+      @admin.setCleanerChoreRunning(java.lang.Boolean::valueOf(enableDisable))
+    end
+
+    #----------------------------------------------------------------------------------------------
+    # Query on the cleaner chore state (enabled/disabled?)
+    # Returns cleaner state (true signifies enabled).
+    def cleaner_chore_enabled()
+      @admin.isCleanerChoreEnabled()
+    end
+
+    #----------------------------------------------------------------------------------------------
     # Enables a table
     def enable(table_name)
       tableExists(table_name)

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b1f57b/hbase-shell/src/main/ruby/shell.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb
index 2c9ab72..5ead8ff 100644
--- a/hbase-shell/src/main/ruby/shell.rb
+++ b/hbase-shell/src/main/ruby/shell.rb
@@ -348,6 +348,9 @@ Shell.load_command_group(
     catalogjanitor_run
     catalogjanitor_switch
     catalogjanitor_enabled
+    cleaner_chore_run
+    cleaner_chore_switch
+    cleaner_chore_enabled
     compact_rs
     compaction_state
     trace

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b1f57b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_enabled.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_enabled.rb b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_enabled.rb
new file mode 100644
index 0000000..6ae4a24
--- /dev/null
+++ b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_enabled.rb
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class CleanerChoreEnabled < Command
+      def help
+        return <<-EOF
+Query for the Cleaner chore state (enabled/disabled?).
+Examples:
+
+  hbase> cleaner_chore_enabled
+EOF
+      end
+
+      def command()
+        formatter.row([admin.cleaner_chore_enabled()? "true" : "false"])
+      end
+    end
+  end
+end
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b1f57b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb
new file mode 100644
index 0000000..9ade9c9
--- /dev/null
+++ b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class CleanerChoreRun < Command
+      def help
+        return <<-EOF
+Cleaner chore command for garbage collection of HFiles and WAL files.
+
+  hbase> cleaner_chore_run
+
+EOF
+      end
+      def command()
+        admin.cleaner_chore_run()
+      end
+    end
+  end
+end
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/f8b1f57b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_switch.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_switch.rb b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_switch.rb
new file mode 100644
index 0000000..20241ec
--- /dev/null
+++ b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_switch.rb
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class CleanerChoreSwitch < Command
+      def help
+        return <<-EOF
+Enable/Disable Cleaner chore. Returns previous Cleaner chore state.
+Examples:
+
+  hbase> cleaner_chore_switch true
+  hbase> cleaner_chore_switch false
+EOF
+      end
+
+      def command(enableDisable)
+        formatter.row([admin.cleaner_chore_switch(enableDisable)? "true" : "false"])
+      end
+    end
+  end
+end
\ No newline at end of file


Mime
View raw message