hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anoopsamj...@apache.org
Subject hbase git commit: HBASE-13734 Improper timestamp checking with VisibilityScanDeleteTracker.
Date Mon, 25 May 2015 08:25:32 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 b53db01df -> 3b2677fe5


HBASE-13734 Improper timestamp checking with VisibilityScanDeleteTracker.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b2677fe
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b2677fe
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b2677fe

Branch: refs/heads/branch-1.1
Commit: 3b2677fe566f23cde0d0c28daeec8f6639b0cd5e
Parents: b53db01
Author: anoopsjohn <anoopsamjohn@gmail.com>
Authored: Mon May 25 13:55:07 2015 +0530
Committer: anoopsjohn <anoopsamjohn@gmail.com>
Committed: Mon May 25 13:55:07 2015 +0530

----------------------------------------------------------------------
 .../visibility/VisibilityScanDeleteTracker.java |  86 ++++++++-------
 .../TestVisibilityLabelsWithDeletes.java        | 106 +++++++++++++++++++
 2 files changed, 153 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/3b2677fe/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java
index 80e1d5d..ea7ce33 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java
@@ -20,12 +20,7 @@ package org.apache.hadoop.hbase.security.visibility;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -37,6 +32,7 @@ import org.apache.hadoop.hbase.Tag;
 import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Triple;
 
 /**
  * Similar to ScanDeletTracker but tracks the visibility expression also before
@@ -55,12 +51,12 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
   // type would solve this problem and also ensure that the combination of different type
   // of deletes with diff ts would also work fine
   // Track per TS
-  private Map<Long, Pair<List<Tag>, Byte>> visibilityTagsDeleteFamily =
-      new HashMap<Long, Pair<List<Tag>, Byte>>();
+  private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamily
=
+      new ArrayList<Triple<List<Tag>, Byte, Long>>();
   // Delete family version with different ts and different visibility expression could come.
   // Need to track it per ts.
-  private Map<Long,Pair<List<Tag>, Byte>> visibilityTagsDeleteFamilyVersion
=
-      new HashMap<Long, Pair<List<Tag>, Byte>>();
+  private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamilyVersion
=
+      new ArrayList<Triple<List<Tag>, Byte, Long>>();
   private List<Pair<List<Tag>, Byte>> visibilityTagsDeleteColumns;
   // Tracking as List<List> is to handle same ts cell but different visibility tag.

   // TODO : Need to handle puts with same ts but different vis tags.
@@ -80,8 +76,10 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
     byte type = delCell.getTypeByte();
     if (type == KeyValue.Type.DeleteFamily.getCode()) {
       hasFamilyStamp = true;
-      //familyStamps.add(delCell.getTimestamp());
-      extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
+      boolean hasVisTag = extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
+      if (!hasVisTag && timestamp > familyStamp) {
+        familyStamp = timestamp;
+      }
       return;
     } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
       familyVersionStamps.add(timestamp);
@@ -115,8 +113,9 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
     extractDeleteCellVisTags(delCell, KeyValue.Type.codeToType(type));
   }
 
-  private void extractDeleteCellVisTags(Cell delCell, Type type) {
+  private boolean extractDeleteCellVisTags(Cell delCell, Type type) {
     // If tag is present in the delete
+    boolean hasVisTag = false;
     if (delCell.getTagsLength() > 0) {
       switch (type) {
       case DeleteFamily:
@@ -124,8 +123,9 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
         if (visibilityTagsDeleteFamily != null) {
           Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
           if (!delTags.isEmpty()) {
-            visibilityTagsDeleteFamily.put(delCell.getTimestamp(), new Pair<List<Tag>,
Byte>(
-                delTags, deleteCellVisTagsFormat));
+            visibilityTagsDeleteFamily.add(new Triple<List<Tag>, Byte, Long>(
+                delTags, deleteCellVisTagsFormat, delCell.getTimestamp()));
+            hasVisTag = true;
           }
         }
         break;
@@ -133,8 +133,9 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
         delTags = new ArrayList<Tag>();
         Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
         if (!delTags.isEmpty()) {
-          visibilityTagsDeleteFamilyVersion.put(delCell.getTimestamp(), new Pair<List<Tag>,
Byte>(
-              delTags, deleteCellVisTagsFormat));
+          visibilityTagsDeleteFamilyVersion.add(new Triple<List<Tag>, Byte, Long>(delTags,
+              deleteCellVisTagsFormat, delCell.getTimestamp()));
+          hasVisTag = true;
         }
         break;
       case DeleteColumn:
@@ -146,6 +147,7 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
         if (!delTags.isEmpty()) {
           visibilityTagsDeleteColumns.add(new Pair<List<Tag>, Byte>(delTags,
               deleteCellVisTagsFormat));
+          hasVisTag = true;
         }
         break;
       case Delete:
@@ -157,6 +159,7 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
         if (!delTags.isEmpty()) {
           visiblityTagsDeleteColumnVersion.add(new Pair<List<Tag>, Byte>(delTags,
               deleteCellVisTagsFormat));
+          hasVisTag = true;
         }
         break;
       default:
@@ -180,6 +183,7 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
         throw new IllegalArgumentException("Invalid delete type");
       }
     }
+    return hasVisTag;
   }
 
   @Override
@@ -190,26 +194,28 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
     try {
       if (hasFamilyStamp) {
         if (visibilityTagsDeleteFamily != null) {
-          Set<Entry<Long, Pair<List<Tag>, Byte>>> deleteFamilies
= visibilityTagsDeleteFamily
-              .entrySet();
-          Iterator<Entry<Long, Pair<List<Tag>, Byte>>> iterator =
deleteFamilies.iterator();
-          while (iterator.hasNext()) {
-            Entry<Long, Pair<List<Tag>, Byte>> entry = iterator.next();
-            if (timestamp <= entry.getKey()) {
+          for (int i = 0; i < visibilityTagsDeleteFamily.size(); i++) {
+            // visibilityTagsDeleteFamily is ArrayList
+            Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamily.get(i);
+            if (timestamp <= triple.getThird()) {
               List<Tag> putVisTags = new ArrayList<Tag>();
               Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
               boolean matchFound = VisibilityLabelServiceManager
                   .getInstance()
                   .getVisibilityLabelService()
-                  .matchVisibility(putVisTags, putCellVisTagsFormat, entry.getValue().getFirst(),
-                      entry.getValue().getSecond());
+                  .matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(),
+                      triple.getSecond());
               if (matchFound) {
+                // A return type of FAMILY_DELETED will cause skip for all remaining cells
from this
+                // family. We would like to match visibility expression on every put cells
after
+                // this and only remove those matching with the family delete visibility.
So we are
+                // returning FAMILY_VERSION_DELETED from here.
                 return DeleteResult.FAMILY_VERSION_DELETED;
               }
             }
           }
         } else {
-          if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
+          if (!VisibilityUtils.isVisibilityTagsPresent(cell) && timestamp<=familyStamp)
{
             // No tags
             return DeleteResult.FAMILY_VERSION_DELETED;
           }
@@ -217,18 +223,20 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
       }
       if (familyVersionStamps.contains(Long.valueOf(timestamp))) {
         if (visibilityTagsDeleteFamilyVersion != null) {
-          Pair<List<Tag>, Byte> tags = visibilityTagsDeleteFamilyVersion.get(Long
-              .valueOf(timestamp));
-          if (tags != null) {
-            List<Tag> putVisTags = new ArrayList<Tag>();
-            Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
-            boolean matchFound = VisibilityLabelServiceManager
-                .getInstance()
-                .getVisibilityLabelService()
-                .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
-                    tags.getSecond());
-            if (matchFound) {
-              return DeleteResult.FAMILY_VERSION_DELETED;
+          for (int i = 0; i < visibilityTagsDeleteFamilyVersion.size(); i++) {
+            // visibilityTagsDeleteFamilyVersion is ArrayList
+            Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamilyVersion.get(i);
+            if (timestamp == triple.getThird()) {
+              List<Tag> putVisTags = new ArrayList<Tag>();
+              Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
+              boolean matchFound = VisibilityLabelServiceManager
+                  .getInstance()
+                  .getVisibilityLabelService()
+                  .matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(),
+                      triple.getSecond());
+              if (matchFound) {
+                return DeleteResult.FAMILY_VERSION_DELETED;
+              }
             }
           }
         } else {
@@ -309,8 +317,8 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
   public void reset() {
     super.reset();
     visibilityTagsDeleteColumns = null;
-    visibilityTagsDeleteFamily = new HashMap<Long, Pair<List<Tag>, Byte>>();
-    visibilityTagsDeleteFamilyVersion = new HashMap<Long, Pair<List<Tag>, Byte>>();
+    visibilityTagsDeleteFamily = new ArrayList<Triple<List<Tag>, Byte, Long>>();
+    visibilityTagsDeleteFamilyVersion = new ArrayList<Triple<List<Tag>, Byte,
Long>>();
     visiblityTagsDeleteColumnVersion = null;
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/3b2677fe/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
index 0430525..1e90866 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
@@ -2733,6 +2734,111 @@ public class TestVisibilityLabelsWithDeletes {
     }
   }
 
+  @Test
+  public void testDeleteWithNoVisibilitiesForPutsAndDeletes() throws Exception {
+    final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+    Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
+    HColumnDescriptor colDesc = new HColumnDescriptor(fam);
+    colDesc.setMaxVersions(5);
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    desc.addFamily(colDesc);
+    hBaseAdmin.createTable(desc);
+    Put p = new Put (Bytes.toBytes("row1"));
+    p.addColumn(fam, qual, value);
+    Table table = TEST_UTIL.getConnection().getTable(tableName);
+    table.put(p);
+    p = new Put (Bytes.toBytes("row1"));
+    p.addColumn(fam, qual1, value);
+    table.put(p);
+    p = new Put (Bytes.toBytes("row2"));
+    p.addColumn(fam, qual, value);
+    table.put(p);
+    p = new Put (Bytes.toBytes("row2"));
+    p.addColumn(fam, qual1, value);
+    table.put(p);
+    Delete d = new Delete(Bytes.toBytes("row1"));
+    table.delete(d);
+    Get g = new Get(Bytes.toBytes("row1"));
+    g.setMaxVersions();
+    g.setAuthorizations(new Authorizations(SECRET, PRIVATE));
+    Result result = table.get(g);
+    assertEquals(0, result.rawCells().length);
+
+    p = new Put (Bytes.toBytes("row1"));
+    p.addColumn(fam, qual, value);
+    table.put(p);
+    result = table.get(g);
+    assertEquals(1, result.rawCells().length);
+  }
+
+  @Test
+  public void testDeleteWithFamilyDeletesOfSameTsButDifferentVisibilities() throws Exception
{
+    final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+    Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
+    HColumnDescriptor colDesc = new HColumnDescriptor(fam);
+    colDesc.setMaxVersions(5);
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    desc.addFamily(colDesc);
+    hBaseAdmin.createTable(desc);
+    Table table = TEST_UTIL.getConnection().getTable(tableName);
+    long t1 = 1234L;
+    CellVisibility cellVisibility1 = new CellVisibility(SECRET);
+    CellVisibility cellVisibility2 = new CellVisibility(PRIVATE);
+    // Cell row1:info:qual:1234 with visibility SECRET
+    Put p = new Put(row1);
+    p.addColumn(fam, qual, t1, value);
+    p.setCellVisibility(cellVisibility1);
+    table.put(p);
+
+    // Cell row1:info:qual1:1234 with visibility PRIVATE
+    p = new Put(row1);
+    p.addColumn(fam, qual1, t1, value);
+    p.setCellVisibility(cellVisibility2);
+    table.put(p);
+
+    Delete d = new Delete(row1);
+    d.addFamily(fam, t1);
+    d.setCellVisibility(cellVisibility2);
+    table.delete(d);
+    d = new Delete(row1);
+    d.addFamily(fam, t1);
+    d.setCellVisibility(cellVisibility1);
+    table.delete(d);
+
+    Get g = new Get(row1);
+    g.setMaxVersions();
+    g.setAuthorizations(new Authorizations(SECRET, PRIVATE));
+    Result result = table.get(g);
+    assertEquals(0, result.rawCells().length);
+
+    // Cell row2:info:qual:1234 with visibility SECRET
+    p = new Put(row2);
+    p.addColumn(fam, qual, t1, value);
+    p.setCellVisibility(cellVisibility1);
+    table.put(p);
+
+    // Cell row2:info:qual1:1234 with visibility PRIVATE
+    p = new Put(row2);
+    p.addColumn(fam, qual1, t1, value);
+    p.setCellVisibility(cellVisibility2);
+    table.put(p);
+
+    d = new Delete(row2);
+    d.addFamilyVersion(fam, t1);
+    d.setCellVisibility(cellVisibility2);
+    table.delete(d);
+    d = new Delete(row2);
+    d.addFamilyVersion(fam, t1);
+    d.setCellVisibility(cellVisibility1);
+    table.delete(d);
+
+    g = new Get(row2);
+    g.setMaxVersions();
+    g.setAuthorizations(new Authorizations(SECRET, PRIVATE));
+    result = table.get(g);
+    assertEquals(0, result.rawCells().length);
+  }
+
   private void scanAll(Result[] next) throws IOException {
     CellScanner cellScanner = next[0].cellScanner();
     cellScanner.advance();


Mime
View raw message