hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [36/54] [abbrv] hbase git commit: HBASE-17025 Add shell commands for space quotas
Date Tue, 11 Apr 2017 17:31:41 GMT
HBASE-17025 Add shell commands for space quotas


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bae29b64
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bae29b64
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bae29b64

Branch: refs/heads/HBASE-16961
Commit: bae29b64a8542f16f392b166ae78ebf3026698dd
Parents: 79bf188
Author: Josh Elser <elserj@apache.org>
Authored: Wed Jan 11 11:55:29 2017 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Tue Apr 11 13:10:02 2017 -0400

----------------------------------------------------------------------
 hbase-shell/src/main/ruby/hbase/quotas.rb       |  62 ++++++++-
 hbase-shell/src/main/ruby/hbase_constants.rb    |   1 +
 .../src/main/ruby/shell/commands/set_quota.rb   |  45 +++++-
 .../hadoop/hbase/client/AbstractTestShell.java  |   1 +
 hbase-shell/src/test/ruby/hbase/quotas_test.rb  | 137 +++++++++++++++++++
 hbase-shell/src/test/ruby/tests_runner.rb       |   1 +
 6 files changed, 242 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/bae29b64/hbase-shell/src/main/ruby/hbase/quotas.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb b/hbase-shell/src/main/ruby/hbase/quotas.rb
index bf2dc63..d99fe72 100644
--- a/hbase-shell/src/main/ruby/hbase/quotas.rb
+++ b/hbase-shell/src/main/ruby/hbase/quotas.rb
@@ -24,14 +24,22 @@ java_import org.apache.hadoop.hbase.quotas.ThrottleType
 java_import org.apache.hadoop.hbase.quotas.QuotaFilter
 java_import org.apache.hadoop.hbase.quotas.QuotaRetriever
 java_import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory
+java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy
 
 module HBaseQuotasConstants
+  # RPC Quota constants
   GLOBAL_BYPASS = 'GLOBAL_BYPASS'
   THROTTLE_TYPE = 'THROTTLE_TYPE'
   THROTTLE = 'THROTTLE'
   REQUEST = 'REQUEST'
   WRITE = 'WRITE'
   READ = 'READ'
+  # Space quota constants
+  SPACE = 'SPACE'
+  NO_INSERTS = 'NO_INSERTS'
+  NO_WRITES = 'NO_WRITES'
+  NO_WRITES_COMPACTIONS = 'NO_WRITES_COMPACTIONS'
+  DISABLE = 'DISABLE'
 end
 
 module Hbase
@@ -107,6 +115,54 @@ module Hbase
       @admin.setQuota(settings)
     end
 
+    def limit_space(args)
+      raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil? and args.kind_of?(Hash))
+      # Let the user provide a raw number
+      if args[LIMIT].is_a?(Numeric)
+        limit = args[LIMIT]
+      else
+        # Parse a string a 1K, 2G, etc.
+        limit = _parse_size(args[LIMIT])
+      end
+      # Extract the policy, failing if something bogus was provided
+      policy = SpaceViolationPolicy.valueOf(args[POLICY])
+      # Create a table or namespace quota
+      if args.key?(TABLE)
+        if args.key?(NAMESPACE)
+          raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.")
+        end
+        settings = QuotaSettingsFactory.limitTableSpace(TableName.valueOf(args.delete(TABLE)),
limit, policy)
+      elsif args.key?(NAMESPACE)
+        if args.key?(TABLE)
+          raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.")
+        end
+        settings = QuotaSettingsFactory.limitNamespaceSpace(args.delete(NAMESPACE), limit,
policy)
+      else
+        raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
+      end
+      # Apply the quota
+      @admin.setQuota(settings)
+    end
+
+    def remove_space_limit(args)
+      raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil? and args.kind_of?(Hash))
+      if args.key?(TABLE)
+        if args.key?(NAMESPACE)
+          raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.")
+        end
+        table = TableName.valueOf(args.delete(TABLE))
+        settings = QuotaSettingsFactory.removeTableSpaceLimit(table)
+      elsif args.key?(NAMESPACE)
+        if args.key?(TABLE)
+          raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.")
+        end
+        settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(args.delete(NAMESPACE))
+      else
+        raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
+      end
+      @admin.setQuota(settings)
+    end
+
     def set_global_bypass(bypass, args)
       raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash)
 
@@ -171,7 +227,7 @@ module Hbase
           return _size_from_str(match[1].to_i, match[2])
         end
       else
-        raise "Invalid size limit syntax"
+        raise(ArgumentError, "Invalid size limit syntax")
       end
     end
 
@@ -188,7 +244,7 @@ module Hbase
         end
 
         if limit <= 0
-          raise "Invalid throttle limit, must be greater then 0"
+          raise(ArgumentError, "Invalid throttle limit, must be greater then 0")
         end
 
         case match[3]
@@ -200,7 +256,7 @@ module Hbase
 
         return type, limit, time_unit
       else
-        raise "Invalid throttle limit syntax"
+        raise(ArgumentError, "Invalid throttle limit syntax")
       end
     end
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/bae29b64/hbase-shell/src/main/ruby/hbase_constants.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase_constants.rb b/hbase-shell/src/main/ruby/hbase_constants.rb
index c02d5c6..a857403 100644
--- a/hbase-shell/src/main/ruby/hbase_constants.rb
+++ b/hbase-shell/src/main/ruby/hbase_constants.rb
@@ -81,6 +81,7 @@ module HBaseConstants
   NAMESPACES = 'NAMESPACES'
   CONFIG = 'CONFIG'
   DATA = 'DATA'
+  POLICY = 'POLICY'
 
   # Load constants from hbase java API
   def self.promote_constants(constants)

http://git-wip-us.apache.org/repos/asf/hbase/blob/bae29b64/hbase-shell/src/main/ruby/shell/commands/set_quota.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/set_quota.rb b/hbase-shell/src/main/ruby/shell/commands/set_quota.rb
index a638b93..06ed0ba 100644
--- a/hbase-shell/src/main/ruby/shell/commands/set_quota.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/set_quota.rb
@@ -52,6 +52,37 @@ For example:
     hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE, USER => 'u1',
LIMIT => NONE
 
     hbase> set_quota USER => 'u1', GLOBAL_BYPASS => true
+
+TYPE => SPACE
+Users can either set a quota on a table or a namespace. The quota is a limit on the target's
+size on the FileSystem and some action to take when the target exceeds that limit. The limit
+is in bytes and can expressed using standard metric suffixes (B, K, M, G, T, P), defaulting
+to bytes if not provided. Different quotas can be applied to one table at the table and namespace
+level; table-level quotas take priority over namespace-level quotas.
+
+There are a limited number of policies to take when a quota is violation, listed in order
of
+least strict to most strict.
+
+  NO_INSERTS - No new data is allowed to be ingested (e.g. Put, Increment, Append).
+  NO_WRITES - Same as NO_INSERTS but Deletes are also disallowed.
+  NO_WRITES_COMPACTIONS - Same as NO_WRITES but compactions are also disallowed.
+  DISABLE - The table(s) are disabled.
+
+For example:
+
+  hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => '1G', POLICY =>
NO_INSERTS
+  hbase> set_quota TYPE => SPACE, TABLE => 't2', LIMIT => '50G', POLICY =>
DISABLE
+  hbase> set_quota TYPE => SPACE, TABLE => 't3', LIMIT => '2T', POLICY =>
NO_WRITES_COMPACTIONS
+  hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '50T', POLICY
=> NO_WRITES
+
+Space quotas can also be removed via this command. To remove a space quota, provide NONE
+for the limit.
+
+For example:
+
+  hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => NONE
+  hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => NONE
+
 EOF
       end
 
@@ -66,8 +97,18 @@ EOF
               else
                 quotas_admin.throttle(args)
               end
-          else
-            raise "Invalid TYPE argument. got " + qtype
+            when SPACE
+              if args[LIMIT].eql? NONE
+                args.delete(LIMIT)
+                # Table/Namespace argument is verified in remove_space_limit
+                quotas_admin.remove_space_limit(args)
+              else
+                raise(ArgumentError, 'Expected a LIMIT to be provided') unless args.key?(LIMIT)
+                raise(ArgumentError, 'Expected a POLICY to be provided') unless args.key?(POLICY)
+                quotas_admin.limit_space(args)
+              end
+            else
+              raise "Invalid TYPE argument. got " + qtype
           end
         elsif args.has_key?(GLOBAL_BYPASS)
           quotas_admin.set_global_bypass(args.delete(GLOBAL_BYPASS), args)

http://git-wip-us.apache.org/repos/asf/hbase/blob/bae29b64/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
index 1403805..53606e9 100644
--- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
+++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
@@ -39,6 +39,7 @@ public abstract class AbstractTestShell {
     // Start mini cluster
     TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
     TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
+    TEST_UTIL.getConfiguration().setBoolean("hbase.quota.enabled", true);
     TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
     TEST_UTIL.getConfiguration().setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false);
     TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);

http://git-wip-us.apache.org/repos/asf/hbase/blob/bae29b64/hbase-shell/src/test/ruby/hbase/quotas_test.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/ruby/hbase/quotas_test.rb b/hbase-shell/src/test/ruby/hbase/quotas_test.rb
new file mode 100644
index 0000000..78c889c
--- /dev/null
+++ b/hbase-shell/src/test/ruby/hbase/quotas_test.rb
@@ -0,0 +1,137 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'shell'
+require 'stringio'
+require 'hbase_constants'
+require 'hbase/hbase'
+require 'hbase/table'
+
+include HBaseConstants
+
+module Hbase
+  class SpaceQuotasTest < Test::Unit::TestCase
+    include TestHelpers
+
+    def setup
+      setup_hbase
+      # Create test table if it does not exist
+      @test_name = "hbase_shell_tests_table"
+      create_test_table(@test_name)
+    end
+
+    def teardown
+      shutdown
+    end
+
+    define_test 'limit_space errors on non Hash argument' do
+      qa = quotas_admin()
+      assert_raise(ArgumentError) do
+        qa.limit_space('foo')
+      end
+      assert_raise(ArgumentError) do
+        qa.limit_space()
+      end
+    end
+
+    define_test 'remove_space_limit errors on non Hash argument' do
+      qa = quotas_admin()
+      assert_raise(ArgumentError) do
+        qa.remove_space_limit('foo')
+      end
+      assert_raise(ArgumentError) do
+        qa.remove_space_limit()
+      end
+    end
+
+    define_test 'set quota with a non-numeric limit fails' do
+      assert_raise(ArgumentError) do
+        command(:set_quota, TYPE => SPACE, LIMIT => 'asdf', POLICY => NO_INSERTS,
TABLE => @test_name)
+      end
+    end
+
+    define_test 'set quota without a limit fails' do
+      assert_raise(ArgumentError) do
+        command(:set_quota, TYPE => SPACE, POLICY => NO_INSERTS, TABLE => @test_name)
+      end
+    end
+
+    define_test 'set quota without a policy fails' do
+      assert_raise(ArgumentError) do
+        command(:set_quota, TYPE => SPACE, LIMIT => '1G', TABLE => @test_name)
+      end
+    end
+
+    define_test 'set quota without a table or namespace fails' do
+      assert_raise(ArgumentError) do
+        command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS)
+      end
+    end
+
+    define_test 'invalid violation policy specified' do
+      assert_raise(NameError) do
+        command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => FOO_BAR, TABLE
=> @test_name)
+      end
+    end
+
+    define_test 'table and namespace are mutually exclusive in set quota' do
+      assert_raise(ArgumentError) do
+        command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS,
TABLE => @test_name, NAMESPACE => "foo")
+      end
+    end
+
+    define_test '_parse_size accepts various forms of byte shorthand' do
+      qa = quotas_admin()
+      KILO = 1024
+      MEGA = KILO * KILO
+      GIGA = MEGA * KILO
+      TERA = GIGA * KILO
+      PETA = TERA * KILO
+      assert_equal(1, qa._parse_size("1"))
+      assert_equal(1, qa._parse_size("1b"))
+      assert_equal(1, qa._parse_size("1B"))
+      assert_equal(KILO * 2, qa._parse_size("2k"))
+      assert_equal(KILO * 2, qa._parse_size("2K"))
+      assert_equal(MEGA * 5, qa._parse_size("5m"))
+      assert_equal(MEGA * 5, qa._parse_size("5M"))
+      assert_equal(GIGA * 3, qa._parse_size("3g"))
+      assert_equal(GIGA * 3, qa._parse_size("3G"))
+      assert_equal(TERA * 4, qa._parse_size("4t"))
+      assert_equal(TERA * 4, qa._parse_size("4T"))
+      assert_equal(PETA * 32, qa._parse_size("32p"))
+      assert_equal(PETA * 32, qa._parse_size("32P"))
+      assert_equal(GIGA * 4, qa._parse_size("4096m"))
+      assert_equal(GIGA * 4, qa._parse_size("4096M"))
+    end
+
+    define_test 'can set and remove quota' do
+      command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS, TABLE
=> @test_name)
+      output = capture_stdout{ command(:list_quotas) }
+      size = 1024 * 1024 * 1024
+      assert(output.include?("LIMIT => #{size}"))
+      assert(output.include?("VIOLATION_POLICY => NO_INSERTS"))
+      assert(output.include?("TYPE => SPACE"))
+      assert(output.include?("TABLE => #{@test_name}"))
+
+      command(:set_quota, TYPE => SPACE, LIMIT => NONE, TABLE => @test_name)
+      output = capture_stdout{ command(:list_quotas) }
+      assert(output.include?("0 row(s)"))
+    end
+  end
+end

http://git-wip-us.apache.org/repos/asf/hbase/blob/bae29b64/hbase-shell/src/test/ruby/tests_runner.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/ruby/tests_runner.rb b/hbase-shell/src/test/ruby/tests_runner.rb
index 74ddb48..54bf3f9 100644
--- a/hbase-shell/src/test/ruby/tests_runner.rb
+++ b/hbase-shell/src/test/ruby/tests_runner.rb
@@ -36,6 +36,7 @@ unless defined?($TEST_CLUSTER)
   $TEST_CLUSTER = HBaseTestingUtility.new
   $TEST_CLUSTER.configuration.setInt("hbase.regionserver.msginterval", 100)
   $TEST_CLUSTER.configuration.setInt("hbase.client.pause", 250)
+  $TEST_CLUSTER.configuration.set("hbase.quota.enabled", "true")
   $TEST_CLUSTER.configuration.setInt(org.apache.hadoop.hbase.HConstants::HBASE_CLIENT_RETRIES_NUMBER,
6)
   $TEST_CLUSTER.startMiniCluster
   @own_cluster = true


Mime
View raw message