Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 5C06B200D06 for ; Sun, 20 Aug 2017 23:30:14 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 5A6CC164117; Sun, 20 Aug 2017 21:30:14 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3B71716410E for ; Sun, 20 Aug 2017 23:30:13 +0200 (CEST) Received: (qmail 55202 invoked by uid 500); 20 Aug 2017 21:30:05 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 51347 invoked by uid 99); 20 Aug 2017 21:30:03 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 20 Aug 2017 21:30:02 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D288DF5F0D; Sun, 20 Aug 2017 21:29:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: busbey@apache.org To: commits@hbase.apache.org Date: Sun, 20 Aug 2017 21:30:42 -0000 Message-Id: <504b602d10194672931756579c04e485@git.apache.org> In-Reply-To: <13059aa551a342019db3831c44c4c7d5@git.apache.org> References: <13059aa551a342019db3831c44c4c7d5@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [45/50] [abbrv] hbase git commit: HBASE-18572 Delete can't remove the cells which have no visibility label archived-at: Sun, 20 Aug 2017 21:30:14 -0000 HBASE-18572 Delete can't remove the cells which have no visibility label Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e9bafeb0 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e9bafeb0 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e9bafeb0 Branch: refs/heads/HBASE-18467 Commit: e9bafeb091549cce35f25a56688b6632578de74b Parents: e2532ec Author: Chia-Ping Tsai Authored: Sat Aug 19 01:55:45 2017 +0800 Committer: Chia-Ping Tsai Committed: Sat Aug 19 01:55:45 2017 +0800 ---------------------------------------------------------------------- .../DefaultVisibilityLabelServiceImpl.java | 12 +- .../visibility/VisibilityController.java | 4 + .../visibility/VisibilityScanDeleteTracker.java | 19 +- .../ExpAsStringVisibilityLabelServiceImpl.java | 4 + .../TestVisibilityLabelsWithDeletes.java | 191 +++++++++++++++++++ 5 files changed, 222 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/e9bafeb0/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java index d4a5627..672da8a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java @@ -551,12 +551,16 @@ public class DefaultVisibilityLabelServiceImpl implements VisibilityLabelService @Override public boolean matchVisibility(List putVisTags, Byte putTagsFormat, List deleteVisTags, Byte deleteTagsFormat) throws IOException { + // Early out if there are no tags in both of cell and delete + if (putVisTags.isEmpty() && deleteVisTags.isEmpty()) { + return true; + } + // Early out if one of the tags is empty + if (putVisTags.isEmpty() ^ deleteVisTags.isEmpty()) { + return false; + } if ((deleteTagsFormat != null && deleteTagsFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT) && (putTagsFormat == null || putTagsFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT)) { - if (putVisTags.isEmpty()) { - // Early out if there are no tags in the cell - return false; - } if (putTagsFormat == null) { return matchUnSortedVisibilityTags(putVisTags, deleteVisTags); } else { http://git-wip-us.apache.org/repos/asf/hbase/blob/e9bafeb0/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java index 23f0583..130587a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java @@ -1074,6 +1074,10 @@ public class VisibilityController implements MasterObserver, RegionObserver, public ReturnCode filterKeyValue(Cell cell) throws IOException { List putVisTags = new ArrayList<>(); Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags); + if (putVisTags.isEmpty() && deleteCellVisTags.isEmpty()) { + // Early out if there are no tags in the cell + return ReturnCode.INCLUDE; + } boolean matchFound = VisibilityLabelServiceManager .getInstance().getVisibilityLabelService() .matchVisibility(putVisTags, putCellVisTagsFormat, deleteCellVisTags, http://git-wip-us.apache.org/repos/asf/hbase/blob/e9bafeb0/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java index 67181e1..f62e8d0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.security.visibility; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.commons.logging.Log; @@ -45,6 +46,10 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker { private static final Log LOG = LogFactory.getLog(VisibilityScanDeleteTracker.class); + /** + * This tag is used for the DELETE cell which has no visibility label. + */ + private static final List EMPTY_TAG = Collections.EMPTY_LIST; // Its better to track the visibility tags in delete based on each type. Create individual // data structures for tracking each of them. This would ensure that there is no tracking based // on time and also would handle all cases where deletefamily or deletecolumns is specified with @@ -110,9 +115,8 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker { private boolean extractDeleteCellVisTags(Cell delCell, Type type) { // If tag is present in the delete boolean hasVisTag = false; - if (delCell.getTagsLength() > 0) { - Byte deleteCellVisTagsFormat = null; - switch (type) { + Byte deleteCellVisTagsFormat = null; + switch (type) { case DeleteFamily: List delTags = new ArrayList<>(); if (visibilityTagsDeleteFamily == null) { @@ -122,6 +126,8 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker { if (!delTags.isEmpty()) { visibilityTagsDeleteFamily.add(new Triple<>(delTags, deleteCellVisTagsFormat, delCell.getTimestamp())); hasVisTag = true; + } else { + visibilityTagsDeleteFamily.add(new Triple<>(EMPTY_TAG, deleteCellVisTagsFormat, delCell.getTimestamp())); } break; case DeleteFamilyVersion: @@ -133,6 +139,8 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker { if (!delTags.isEmpty()) { visibilityTagsDeleteFamilyVersion.add(new Triple<>(delTags, deleteCellVisTagsFormat, delCell.getTimestamp())); hasVisTag = true; + } else { + visibilityTagsDeleteFamilyVersion.add(new Triple<>(EMPTY_TAG, deleteCellVisTagsFormat, delCell.getTimestamp())); } break; case DeleteColumn: @@ -144,6 +152,8 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker { if (!delTags.isEmpty()) { visibilityTagsDeleteColumns.add(new Pair<>(delTags, deleteCellVisTagsFormat)); hasVisTag = true; + } else { + visibilityTagsDeleteColumns.add(new Pair<>(EMPTY_TAG, deleteCellVisTagsFormat)); } break; case Delete: @@ -155,11 +165,12 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker { if (!delTags.isEmpty()) { visiblityTagsDeleteColumnVersion.add(new Pair<>(delTags, deleteCellVisTagsFormat)); hasVisTag = true; + } else { + visiblityTagsDeleteColumnVersion.add(new Pair<>(EMPTY_TAG, deleteCellVisTagsFormat)); } break; default: throw new IllegalArgumentException("Invalid delete type"); - } } return hasVisTag; } http://git-wip-us.apache.org/repos/asf/hbase/blob/e9bafeb0/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/ExpAsStringVisibilityLabelServiceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/ExpAsStringVisibilityLabelServiceImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/ExpAsStringVisibilityLabelServiceImpl.java index d8d6f1e..043d08b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/ExpAsStringVisibilityLabelServiceImpl.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/ExpAsStringVisibilityLabelServiceImpl.java @@ -418,6 +418,10 @@ public class ExpAsStringVisibilityLabelServiceImpl implements VisibilityLabelSer private static boolean checkForMatchingVisibilityTagsWithSortedOrder(List putVisTags, List deleteVisTags) { + // Early out if there are no tags in both of cell and delete + if (putVisTags.isEmpty() && deleteVisTags.isEmpty()) { + return true; + } boolean matchFound = false; // If the size does not match. Definitely we are not comparing the equal // tags. http://git-wip-us.apache.org/repos/asf/hbase/blob/e9bafeb0/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java index dfc48bf..4f48236 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java @@ -17,9 +17,12 @@ */ package org.apache.hadoop.hbase.security.visibility; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; @@ -41,6 +44,9 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.SecurityTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.DefaultEnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Threads; import org.junit.After; import org.junit.AfterClass; @@ -67,6 +73,7 @@ import static org.junit.Assert.assertTrue; */ @Category({SecurityTests.class, MediumTests.class}) public class TestVisibilityLabelsWithDeletes { + private static final Log LOG = LogFactory.getLog(TestVisibilityLabelsWithDeletes.class); private static final String TOPSECRET = "TOPSECRET"; private static final String PUBLIC = "PUBLIC"; private static final String PRIVATE = "PRIVATE"; @@ -3285,4 +3292,188 @@ public class TestVisibilityLabelsWithDeletes { public static List createList(T... ts) { return new ArrayList<>(Arrays.asList(ts)); } + + + private enum DeleteMark { + ROW, + FAMILY, + FAMILY_VERSION, + COLUMN, + CELL + } + + private static Delete addDeleteMark(Delete d, DeleteMark mark, long now) { + switch (mark) { + case ROW: + break; + case FAMILY: + d.addFamily(fam); + break; + case FAMILY_VERSION: + d.addFamilyVersion(fam, now); + break; + case COLUMN: + d.addColumns(fam, qual); + break; + case CELL: + d.addColumn(fam, qual); + break; + default: + break; + } + return d; + } + + @Test + public void testDeleteCellWithoutVisibility() throws IOException, InterruptedException { + for (DeleteMark mark : DeleteMark.values()) { + testDeleteCellWithoutVisibility(mark); + } + } + + private void testDeleteCellWithoutVisibility(DeleteMark mark) throws IOException, InterruptedException { + setAuths(); + final TableName tableName = TableName.valueOf("testDeleteCellWithoutVisibility-" + mark.name()); + Admin hBaseAdmin = TEST_UTIL.getAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + long now = EnvironmentEdgeManager.currentTime(); + List puts = new ArrayList<>(1); + Put put = new Put(row1); + if (mark == DeleteMark.FAMILY_VERSION) { + put.addColumn(fam, qual, now, value); + } else { + put.addColumn(fam, qual, value); + } + + puts.add(put); + try (Table table = TEST_UTIL.getConnection().getTable(tableName)){ + table.put(puts); + Result r = table.get(new Get(row1)); + assertEquals(1, r.size()); + assertEquals(Bytes.toString(value), Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); + + Delete d = addDeleteMark(new Delete(row1), mark, now); + table.delete(d); + r = table.get(new Get(row1)); + assertEquals(0, r.size()); + } + } + + @Test + public void testDeleteCellWithVisibility() throws IOException, InterruptedException { + for (DeleteMark mark : DeleteMark.values()) { + testDeleteCellWithVisibility(mark); + testDeleteCellWithVisibilityV2(mark); + } + } + + private void testDeleteCellWithVisibility(DeleteMark mark) throws IOException, InterruptedException { + setAuths(); + final TableName tableName = TableName.valueOf("testDeleteCellWithVisibility-" + mark.name()); + Admin hBaseAdmin = TEST_UTIL.getAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + long now = EnvironmentEdgeManager.currentTime(); + List puts = new ArrayList<>(2); + Put put = new Put(row1); + if (mark == DeleteMark.FAMILY_VERSION) { + put.addColumn(fam, qual, now, value); + } else { + put.addColumn(fam, qual, value); + } + puts.add(put); + put = new Put(row1); + if (mark == DeleteMark.FAMILY_VERSION) { + put.addColumn(fam, qual, now, value1); + } else { + put.addColumn(fam, qual, value1); + } + put.setCellVisibility(new CellVisibility(PRIVATE)); + puts.add(put); + try (Table table = TEST_UTIL.getConnection().getTable(tableName)) { + table.put(puts); + Result r = table.get(new Get(row1)); + assertEquals(0, r.size()); + r = table.get(new Get(row1).setAuthorizations(new Authorizations(PRIVATE))); + assertEquals(1, r.size()); + assertEquals(Bytes.toString(value1), Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); + + Delete d = addDeleteMark(new Delete(row1), mark, now); + table.delete(d); + + r = table.get(new Get(row1)); + assertEquals(0, r.size()); + r = table.get(new Get(row1).setAuthorizations(new Authorizations(PRIVATE))); + assertEquals(1, r.size()); + assertEquals(Bytes.toString(value1), Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); + + d = addDeleteMark(new Delete(row1).setCellVisibility(new CellVisibility(PRIVATE)), mark, now); + table.delete(d); + + r = table.get(new Get(row1)); + assertEquals(0, r.size()); + r = table.get(new Get(row1).setAuthorizations(new Authorizations(PRIVATE))); + assertEquals(0, r.size()); + } + } + + private void testDeleteCellWithVisibilityV2(DeleteMark mark) throws IOException, InterruptedException { + setAuths(); + final TableName tableName = TableName.valueOf("testDeleteCellWithVisibilityV2-" + mark.name()); + Admin hBaseAdmin = TEST_UTIL.getAdmin(); + HColumnDescriptor colDesc = new HColumnDescriptor(fam); + colDesc.setMaxVersions(5); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(colDesc); + hBaseAdmin.createTable(desc); + long now = EnvironmentEdgeManager.currentTime(); + List puts = new ArrayList<>(2); + Put put = new Put(row1); + put.setCellVisibility(new CellVisibility(PRIVATE)); + if (mark == DeleteMark.FAMILY_VERSION) { + put.addColumn(fam, qual, now, value); + } else { + put.addColumn(fam, qual, value); + } + puts.add(put); + put = new Put(row1); + if (mark == DeleteMark.FAMILY_VERSION) { + put.addColumn(fam, qual, now, value1); + } else { + put.addColumn(fam, qual, value1); + } + puts.add(put); + try (Table table = TEST_UTIL.getConnection().getTable(tableName)){ + table.put(puts); + Result r = table.get(new Get(row1)); + assertEquals(1, r.size()); + assertEquals(Bytes.toString(value1), Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); + r = table.get(new Get(row1).setAuthorizations(new Authorizations(PRIVATE))); + assertEquals(1, r.size()); + assertEquals(Bytes.toString(value1), Bytes.toString(CellUtil.cloneValue(r.rawCells()[0]))); + + Delete d = addDeleteMark(new Delete(row1), mark, now); + table.delete(d); + + r = table.get(new Get(row1)); + assertEquals(0, r.size()); + r = table.get(new Get(row1).setAuthorizations(new Authorizations(PRIVATE))); + assertEquals(0, r.size()); + + d = addDeleteMark(new Delete(row1).setCellVisibility(new CellVisibility(PRIVATE)), mark, now); + table.delete(d); + + r = table.get(new Get(row1)); + assertEquals(0, r.size()); + r = table.get(new Get(row1).setAuthorizations(new Authorizations(PRIVATE))); + assertEquals(0, r.size()); + } + } }