Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 64B85200C6B for ; Mon, 17 Apr 2017 22:27:14 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 61D28160BB5; Mon, 17 Apr 2017 20:27:14 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3C253160BB0 for ; Mon, 17 Apr 2017 22:27:13 +0200 (CEST) Received: (qmail 1548 invoked by uid 500); 17 Apr 2017 20:27:11 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 916 invoked by uid 99); 17 Apr 2017 20:27:11 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 17 Apr 2017 20:27:11 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 58688DF9FA; Mon, 17 Apr 2017 20:27:11 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@hbase.apache.org Date: Mon, 17 Apr 2017 20:27:37 -0000 Message-Id: <4f9979c14249411d8a86031f789f0438@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [28/50] [abbrv] hbase git commit: HBASE-17001 Enforce quota violation policies in the RegionServer archived-at: Mon, 17 Apr 2017 20:27:14 -0000 http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java new file mode 100644 index 0000000..ec8f1bf --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas.policies; + +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.util.Bytes; + +public class BaseViolationPolicyEnforcement { + + static final Append APPEND = new Append(Bytes.toBytes("foo")); + static final Delete DELETE = new Delete(Bytes.toBytes("foo")); + static final Increment INCREMENT = new Increment(Bytes.toBytes("foo")); + static final Put PUT = new Put(Bytes.toBytes("foo")); +} http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java new file mode 100644 index 0000000..abe1b9d --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas.policies; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.quotas.SpaceLimitingException; +import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; +import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus; +import org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestBulkLoadCheckingViolationPolicyEnforcement { + + FileSystem fs; + RegionServerServices rss; + TableName tableName; + SpaceViolationPolicyEnforcement policy; + + @Before + public void setup() { + fs = mock(FileSystem.class); + rss = mock(RegionServerServices.class); + tableName = TableName.valueOf("foo"); + policy = new BulkLoadVerifyingViolationPolicyEnforcement(); + } + + @Test + public void testFilesUnderLimit() throws Exception { + final List paths = new ArrayList<>(); + final List statuses = new ArrayList<>(); + final long length = 100L * 1024L; + for (int i = 0; i < 5; i++) { + String path = "/" + i; + FileStatus status = mock(FileStatus.class); + when(fs.getFileStatus(new Path(path))).thenReturn(status); + when(status.getLen()).thenReturn(length); + when(status.isFile()).thenReturn(true); + paths.add(path); + statuses.add(status); + } + + // Quota is not in violation now + SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, length * 6); + + policy.initialize(rss, tableName, snapshot); + + policy.checkBulkLoad(fs, paths); + } + + @Test(expected = IllegalArgumentException.class) + public void testFileIsNotAFile() throws Exception { + final List paths = new ArrayList<>(); + String path = "/1"; + FileStatus status = mock(FileStatus.class); + when(fs.getFileStatus(new Path(path))).thenReturn(status); + when(status.getLen()).thenReturn(1000L); + when(status.isFile()).thenReturn(false); + paths.add(path); + + // Quota is not in violation now + SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, Long.MAX_VALUE); + + policy.initialize(rss, tableName, snapshot); + + // If the file to bulk load isn't a file, this should throw an exception + policy.checkBulkLoad(fs, paths); + } + + @Test(expected = SpaceLimitingException.class) + public void testOneFileInBatchOverLimit() throws Exception { + final List paths = new ArrayList<>(); + final List statuses = new ArrayList<>(); + final long length = 1000L * 1024L; + for (int i = 0; i < 5; i++) { + String path = "/" + i; + FileStatus status = mock(FileStatus.class); + when(fs.getFileStatus(new Path(path))).thenReturn(status); + when(status.getLen()).thenReturn(length); + when(status.isFile()).thenReturn(true); + paths.add(path); + statuses.add(status); + } + + // Quota is not in violation now + SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 1024L); + + policy.initialize(rss, tableName, snapshot); + + policy.checkBulkLoad(fs, paths); + } + + @Test(expected = SpaceLimitingException.class) + public void testSumOfFilesOverLimit() throws Exception { + final List paths = new ArrayList<>(); + final List statuses = new ArrayList<>(); + final long length = 1024L; + for (int i = 0; i < 5; i++) { + String path = "/" + i; + FileStatus status = mock(FileStatus.class); + when(fs.getFileStatus(new Path(path))).thenReturn(status); + when(status.getLen()).thenReturn(length); + when(status.isFile()).thenReturn(true); + paths.add(path); + statuses.add(status); + } + + // Quota is not in violation now, but 5*1024 files would push us to violation + SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 5000L); + + policy.initialize(rss, tableName, snapshot); + + policy.checkBulkLoad(fs, paths); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestDisableTableViolationPolicyEnforcement.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestDisableTableViolationPolicyEnforcement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestDisableTableViolationPolicyEnforcement.java new file mode 100644 index 0000000..c42d866 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestDisableTableViolationPolicyEnforcement.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas.policies; + +import org.apache.hadoop.hbase.quotas.SpaceLimitingException; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test class for {@link DisableTableViolationPolicyEnforcement}. + */ +@Category(SmallTests.class) +public class TestDisableTableViolationPolicyEnforcement extends BaseViolationPolicyEnforcement { + + private DisableTableViolationPolicyEnforcement enforcement; + + @Before + public void setup() { + enforcement = new DisableTableViolationPolicyEnforcement(); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckPut() throws SpaceLimitingException { + // If the policy is enacted, it will always throw an exception + // to avoid constantly re-checking the table state. + enforcement.check(PUT); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckAppend() throws SpaceLimitingException { + enforcement.check(APPEND); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckDelete() throws SpaceLimitingException { + enforcement.check(DELETE); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckIncrement() throws SpaceLimitingException { + enforcement.check(INCREMENT); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoInsertsViolationPolicyEnforcement.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoInsertsViolationPolicyEnforcement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoInsertsViolationPolicyEnforcement.java new file mode 100644 index 0000000..1115fab --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoInsertsViolationPolicyEnforcement.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas.policies; + +import org.apache.hadoop.hbase.quotas.SpaceLimitingException; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test class for {@link NoInsertsViolationPolicyEnforcement}. + */ +@Category(SmallTests.class) +public class TestNoInsertsViolationPolicyEnforcement extends BaseViolationPolicyEnforcement { + + private NoInsertsViolationPolicyEnforcement enforcement; + + @Before + public void setup() { + enforcement = new NoInsertsViolationPolicyEnforcement(); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckAppend() throws Exception { + enforcement.check(APPEND); + } + + @Test + public void testCheckDelete() throws Exception { + enforcement.check(DELETE); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckIncrement() throws Exception { + enforcement.check(INCREMENT); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckPut() throws Exception { + enforcement.check(PUT); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesCompactionsViolationPolicyEnforcement.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesCompactionsViolationPolicyEnforcement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesCompactionsViolationPolicyEnforcement.java new file mode 100644 index 0000000..1348eb7 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesCompactionsViolationPolicyEnforcement.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas.policies; + +import org.apache.hadoop.hbase.quotas.SpaceLimitingException; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test class for {@link NoWritesCompactionsViolationPolicyEnforcement}; + */ +@Category(SmallTests.class) +public class TestNoWritesCompactionsViolationPolicyEnforcement + extends BaseViolationPolicyEnforcement { + + private NoWritesCompactionsViolationPolicyEnforcement enforcement; + + @Before + public void setup() { + enforcement = new NoWritesCompactionsViolationPolicyEnforcement(); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckAppend() throws Exception { + enforcement.check(APPEND); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckDelete() throws Exception { + enforcement.check(DELETE); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckIncrement() throws Exception { + enforcement.check(INCREMENT); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckPut() throws Exception { + enforcement.check(PUT); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesViolationPolicyEnforcement.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesViolationPolicyEnforcement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesViolationPolicyEnforcement.java new file mode 100644 index 0000000..c032f26 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestNoWritesViolationPolicyEnforcement.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas.policies; + +import org.apache.hadoop.hbase.quotas.SpaceLimitingException; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test class for {@link NoWritesViolationPolicyEnforcement}. + */ +@Category(SmallTests.class) +public class TestNoWritesViolationPolicyEnforcement extends BaseViolationPolicyEnforcement { + + private NoWritesViolationPolicyEnforcement enforcement; + + @Before + public void setup() { + enforcement = new NoWritesViolationPolicyEnforcement(); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckAppend() throws Exception { + enforcement.check(APPEND); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckDelete() throws Exception { + enforcement.check(DELETE); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckIncrement() throws Exception { + enforcement.check(INCREMENT); + } + + @Test(expected = SpaceLimitingException.class) + public void testCheckPut() throws Exception { + enforcement.check(PUT); + } +}