hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anoopsamj...@apache.org
Subject [3/3] git commit: HBASE-11553 Abstract visibility label related services into an interface.
Date Wed, 20 Aug 2014 09:21:09 GMT
HBASE-11553 Abstract visibility label related services into an interface.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d7986121
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d7986121
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d7986121

Branch: refs/heads/0.98
Commit: d79861213da95df663136ba53930a789c4531e3f
Parents: 2e978a1
Author: anoopsjohn <anoopsamjohn@gmail.com>
Authored: Wed Aug 20 14:50:36 2014 +0530
Committer: anoopsjohn <anoopsamjohn@gmail.com>
Committed: Wed Aug 20 14:50:36 2014 +0530

----------------------------------------------------------------------
 .../visibility/VisibilityConstants.java         |   6 +-
 .../hbase/zookeeper/ZooKeeperListener.java      |   9 +-
 .../java/org/apache/hadoop/hbase/TagType.java   |   2 +-
 .../hadoop/hbase/mapreduce/LabelExpander.java   |   9 +-
 .../hbase/regionserver/OperationStatus.java     |   7 +-
 .../visibility/DefaultScanLabelGenerator.java   |   6 +-
 .../DefaultVisibilityLabelServiceImpl.java      | 698 +++++++++++++++
 .../visibility/EnforcingScanLabelGenerator.java |   6 +-
 .../visibility/VisibilityController.java        | 855 +++++--------------
 .../visibility/VisibilityExpEvaluator.java      |  42 +
 .../visibility/VisibilityLabelFilter.java       |  56 +-
 .../visibility/VisibilityLabelService.java      | 142 +++
 .../VisibilityLabelServiceManager.java          | 102 +++
 .../visibility/VisibilityLabelsCache.java       | 233 +++++
 .../visibility/VisibilityLabelsManager.java     | 198 -----
 .../visibility/VisibilityScanDeleteTracker.java | 215 +++--
 .../security/visibility/VisibilityUtils.java    | 213 ++---
 .../visibility/ZKVisibilityLabelWatcher.java    |  10 +-
 .../ExpAsStringVisibilityLabelServiceImpl.java  | 395 +++++++++
 .../visibility/TestVisibilityLabels.java        | 189 ++--
 ...VisibilityLabelsWithCustomVisLabService.java |  79 ++
 ...ibilityLabelsWithDefaultVisLabelService.java | 142 +++
 ...isibilityLabelsWithDistributedLogReplay.java |   4 +-
 .../TestVisibilityWithCheckAuths.java           |   4 +-
 24 files changed, 2338 insertions(+), 1284 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/d7986121/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java
index d91f0ef..b56f250 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityConstants.java
@@ -44,10 +44,10 @@ public final class VisibilityConstants {
    * Visibility serialization version format. It indicates the visibility labels
    * are sorted based on ordinal
    **/
-  public static final byte VISIBILITY_SERIALIZATION_VERSION = 1;
+  public static final byte SORTED_ORDINAL_SERIALIZATION_FORMAT = 1;
   /** Byte representation of the visibility_serialization_version **/
-  public static final byte[] SORTED_ORDINAL_SERIALIZATION_FORMAT =
-      new byte[] { VISIBILITY_SERIALIZATION_VERSION };
+  public static final byte[] SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG_VAL =
+      new byte[] { SORTED_ORDINAL_SERIALIZATION_FORMAT };
 
   public static final String CHECK_AUTHS_FOR_MUTATION = 
       "hbase.security.visibility.mutations.checkauths";

http://git-wip-us.apache.org/repos/asf/hbase/blob/d7986121/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java
index 56fe348..178b259 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperListener.java
@@ -19,8 +19,6 @@
 package org.apache.hadoop.hbase.zookeeper;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
 
 /**
  * Base class for internal listeners of ZooKeeper events.
@@ -78,4 +76,11 @@ public abstract class ZooKeeperListener {
   public void nodeChildrenChanged(String path) {
     // no-op
   }
+
+  /**
+   * @return The watcher associated with this listener
+   */
+  public ZooKeeperWatcher getWatcher() {
+    return this.watcher;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/d7986121/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java
index 68018c4..a21f7de 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/TagType.java
@@ -27,5 +27,5 @@ public final class TagType {
   public static final byte ACL_TAG_TYPE = (byte) 1;
   public static final byte VISIBILITY_TAG_TYPE = (byte) 2;
   public static final byte LOG_REPLAY_TAG_TYPE = (byte) 3;
-  public static final byte VISIBILITY_EXP_SERIALIZATION_TAG_TYPE = (byte)4;
+  public static final byte VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE = (byte)4;
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/d7986121/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java
index 60a3bc3..e4d4fe9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LabelExpander.java
@@ -33,6 +33,7 @@ import java.util.Map;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TagType;
 import org.apache.hadoop.hbase.KeyValue.Type;
 import org.apache.hadoop.hbase.Tag;
 import org.apache.hadoop.hbase.client.HTable;
@@ -85,11 +86,11 @@ public class LabelExpander {
     // We will be adding this tag before the visibility tags and the presence of
     // this
     // tag indicates we are supporting deletes with cell visibility
-    tags.add(VisibilityUtils.VIS_SERIALIZATION_TAG);
+    tags.add(VisibilityUtils.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG);
     if (node.isSingleNode()) {
       getLabelOrdinals(node, labelOrdinals);
       writeLabelOrdinalsToStream(labelOrdinals, dos);
-      tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()));
+      tags.add(new Tag(TagType.VISIBILITY_TAG_TYPE, baos.toByteArray()));
       baos.reset();
     } else {
       NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node;
@@ -97,14 +98,14 @@ public class LabelExpander {
         for (ExpressionNode child : nlNode.getChildExps()) {
           getLabelOrdinals(child, labelOrdinals);
           writeLabelOrdinalsToStream(labelOrdinals, dos);
-          tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()));
+          tags.add(new Tag(TagType.VISIBILITY_TAG_TYPE, baos.toByteArray()));
           baos.reset();
           labelOrdinals.clear();
         }
       } else {
         getLabelOrdinals(nlNode, labelOrdinals);
         writeLabelOrdinalsToStream(labelOrdinals, dos);
-        tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()));
+        tags.add(new Tag(TagType.VISIBILITY_TAG_TYPE, baos.toByteArray()));
         baos.reset();
       }
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/d7986121/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java
index e78f27a..aa377c9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/OperationStatus.java
@@ -56,7 +56,11 @@ public class OperationStatus {
     this.exceptionMsg = exceptionMsg;
   }
 
-  
+  public OperationStatus(OperationStatusCode code, Exception e) {
+    this.code = code;
+    this.exceptionMsg = (e == null) ? "" : e.getClass().getName() + ": " + e.getMessage();
+  }
+
   /**
    * @return OperationStatusCode
    */
@@ -70,5 +74,4 @@ public class OperationStatus {
   public String getExceptionMsg() {
     return exceptionMsg;
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/d7986121/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultScanLabelGenerator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultScanLabelGenerator.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultScanLabelGenerator.java
index 5e2368b..19477b1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultScanLabelGenerator.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultScanLabelGenerator.java
@@ -38,10 +38,10 @@ public class DefaultScanLabelGenerator implements ScanLabelGenerator {
   
   private Configuration conf;
   
-  private VisibilityLabelsManager labelsManager;
+  private VisibilityLabelsCache labelsCache;
   
   public DefaultScanLabelGenerator() {
-    this.labelsManager = VisibilityLabelsManager.get();
+    this.labelsCache = VisibilityLabelsCache.get();
   }
 
   @Override
@@ -59,7 +59,7 @@ public class DefaultScanLabelGenerator implements ScanLabelGenerator {
     if (authorizations != null) {
       List<String> labels = authorizations.getLabels();
       String userName = user.getShortName();
-      List<String> auths = this.labelsManager.getAuths(userName);
+      List<String> auths = this.labelsCache.getAuths(userName);
       return dropLabelsNotInUserAuths(labels, auths, userName);
     }
     return null;

http://git-wip-us.apache.org/repos/asf/hbase/blob/d7986121/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java
new file mode 100644
index 0000000..243f6f6
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/DefaultVisibilityLabelServiceImpl.java
@@ -0,0 +1,698 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.security.visibility;
+
+import static org.apache.hadoop.hbase.TagType.VISIBILITY_TAG_TYPE;
+import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
+import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
+import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER;
+import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT;
+import static org.apache.hadoop.hbase.security.visibility.VisibilityUtils.SYSTEM_LABEL;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.io.util.StreamUtils;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.OperationStatus;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.security.AccessDeniedException;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.security.access.AccessControlLists;
+import org.apache.hadoop.hbase.security.visibility.expression.ExpressionNode;
+import org.apache.hadoop.hbase.security.visibility.expression.LeafExpressionNode;
+import org.apache.hadoop.hbase.security.visibility.expression.NonLeafExpressionNode;
+import org.apache.hadoop.hbase.security.visibility.expression.Operator;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+
+import com.google.common.collect.Lists;
+
+@InterfaceAudience.Private
+public class DefaultVisibilityLabelServiceImpl implements VisibilityLabelService {
+
+  private static final Log LOG = LogFactory.getLog(DefaultVisibilityLabelServiceImpl.class);
+
+  // "system" label is having an ordinal value 1.
+  private static final int SYSTEM_LABEL_ORDINAL = 1;
+  private static final Tag[] LABELS_TABLE_TAGS = new Tag[1];
+  private static final byte[] DUMMY_VALUE = new byte[0];
+
+  private volatile int ordinalCounter = -1;
+  private final ExpressionParser expressionParser = new ExpressionParser();
+  private final ExpressionExpander expressionExpander = new ExpressionExpander();
+  private Configuration conf;
+  private HRegion labelsRegion;
+  private VisibilityLabelsCache labelsCache;
+  private List<ScanLabelGenerator> scanLabelGenerators;
+
+  static {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    DataOutputStream dos = new DataOutputStream(baos);
+    try {
+      StreamUtils.writeRawVInt32(dos, SYSTEM_LABEL_ORDINAL);
+    } catch (IOException e) {
+      // We write to a byte array. No Exception can happen.
+    }
+    LABELS_TABLE_TAGS[0] = new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray());
+  }
+
+  public DefaultVisibilityLabelServiceImpl() {
+
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  @Override
+  public void init(RegionCoprocessorEnvironment e) throws IOException {
+    ZooKeeperWatcher zk = e.getRegionServerServices().getZooKeeper();
+    try {
+      labelsCache = VisibilityLabelsCache.createAndGet(zk, this.conf);
+    } catch (IOException ioe) {
+      LOG.error("Error creating VisibilityLabelsCache", ioe);
+      throw ioe;
+    }
+    this.scanLabelGenerators = VisibilityUtils.getScanLabelGenerators(this.conf);
+    if (e.getRegion().getRegionInfo().getTable().equals(LABELS_TABLE_NAME)) {
+      this.labelsRegion = e.getRegion();
+      Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths
=
+          extractLabelsAndAuths(getExistingLabelsWithAuths());
+      Map<String, Integer> labels = labelsAndUserAuths.getFirst();
+      Map<String, List<Integer>> userAuths = labelsAndUserAuths.getSecond();
+      // Add the "system" label if it is not added into the system yet
+      addSystemLabel(this.labelsRegion, labels, userAuths);
+      int ordinal = SYSTEM_LABEL_ORDINAL; // Ordinal 1 is reserved for "system" label.
+      for (Integer i : labels.values()) {
+        if (i > ordinal) {
+          ordinal = i;
+        }
+      }
+      this.ordinalCounter = ordinal + 1;
+      if (labels.size() > 0) {
+        // If there is no data need not write to zk
+        byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(labels);
+        this.labelsCache.writeToZookeeper(serialized, true);
+        this.labelsCache.refreshLabelsCache(serialized);
+      }
+      if (userAuths.size() > 0) {
+        byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths);
+        this.labelsCache.writeToZookeeper(serialized, false);
+        this.labelsCache.refreshUserAuthsCache(serialized);
+      }
+    }
+  }
+
+  protected List<List<Cell>> getExistingLabelsWithAuths() throws IOException
{
+    Scan scan = new Scan();
+    RegionScanner scanner = labelsRegion.getScanner(scan);
+    List<List<Cell>> existingLabels = new ArrayList<List<Cell>>();
+    try {
+      while (true) {
+        List<Cell> cells = new ArrayList<Cell>();
+        scanner.next(cells);
+        if (cells.isEmpty()) {
+          break;
+        }
+        existingLabels.add(cells);
+      }
+    } finally {
+      scanner.close();
+    }
+    return existingLabels;
+  }
+
+  protected Pair<Map<String, Integer>, Map<String, List<Integer>>>
extractLabelsAndAuths(
+      List<List<Cell>> labelDetails) {
+    Map<String, Integer> labels = new HashMap<String, Integer>();
+    Map<String, List<Integer>> userAuths = new HashMap<String, List<Integer>>();
+    for (List<Cell> cells : labelDetails) {
+      for (Cell cell : cells) {
+        if (Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
+            cell.getQualifierLength(), LABEL_QUALIFIER, 0, LABEL_QUALIFIER.length)) {
+          labels.put(
+              Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()),
+              Bytes.toInt(cell.getRowArray(), cell.getRowOffset()));
+        } else {
+          // These are user cells who has authorization for this label
+          String user = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(),
+              cell.getQualifierLength());
+          List<Integer> auths = userAuths.get(user);
+          if (auths == null) {
+            auths = new ArrayList<Integer>();
+            userAuths.put(user, auths);
+          }
+          auths.add(Bytes.toInt(cell.getRowArray(), cell.getRowOffset()));
+        }
+      }
+    }
+    return new Pair<Map<String, Integer>, Map<String, List<Integer>>>(labels,
userAuths);
+  }
+
+  protected void addSystemLabel(HRegion region, Map<String, Integer> labels,
+      Map<String, List<Integer>> userAuths) throws IOException {
+    if (!labels.containsKey(SYSTEM_LABEL)) {
+      Put p = new Put(Bytes.toBytes(SYSTEM_LABEL_ORDINAL));
+      p.addImmutable(LABELS_TABLE_FAMILY, LABEL_QUALIFIER, Bytes.toBytes(SYSTEM_LABEL));
+      // Set auth for "system" label for all super users.
+      List<String> superUsers = getSystemAndSuperUsers();
+      for (String superUser : superUsers) {
+        p.addImmutable(LABELS_TABLE_FAMILY, Bytes.toBytes(superUser), DUMMY_VALUE,
+            LABELS_TABLE_TAGS);
+      }
+      region.put(p);
+      labels.put(SYSTEM_LABEL, SYSTEM_LABEL_ORDINAL);
+      for (String superUser : superUsers) {
+        List<Integer> auths = userAuths.get(superUser);
+        if (auths == null) {
+          auths = new ArrayList<Integer>(1);
+          userAuths.put(superUser, auths);
+        }
+        auths.add(SYSTEM_LABEL_ORDINAL);
+      }
+    }
+  }
+
+  protected List<String> getSystemAndSuperUsers() throws IOException {
+    User user = User.getCurrent();
+    if (user == null) {
+      throw new IOException("Unable to obtain the current user, "
+          + "authorization checks for internal operations will not work correctly!");
+    }
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Current user name is " + user.getShortName());
+    }
+    String currentUser = user.getShortName();
+    List<String> superUsers = Lists.asList(currentUser,
+        this.conf.getStrings(AccessControlLists.SUPERUSER_CONF_KEY, new String[0]));
+    return superUsers;
+  }
+
+  @Override
+  public OperationStatus[] addLabels(List<byte[]> labels) throws IOException {
+    assert labelsRegion != null;
+    OperationStatus[] finalOpStatus = new OperationStatus[labels.size()];
+    List<Mutation> puts = new ArrayList<Mutation>(labels.size());
+    int i = 0;
+    for (byte[] label : labels) {
+      String labelStr = Bytes.toString(label);
+      if (this.labelsCache.getLabelOrdinal(labelStr) > 0) {
+        finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE,
+            new LabelAlreadyExistsException("Label '" + labelStr + "' already exists"));
+      } else {
+        Put p = new Put(Bytes.toBytes(ordinalCounter));
+        p.addImmutable(LABELS_TABLE_FAMILY, LABEL_QUALIFIER, label, LABELS_TABLE_TAGS);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Adding the label " + labelStr);
+        }
+        puts.add(p);
+        ordinalCounter++;
+      }
+      i++;
+    }
+    if (mutateLabelsRegion(puts, finalOpStatus)) {
+      updateZk(true);
+    }
+    return finalOpStatus;
+  }
+
+  @Override
+  public OperationStatus[] setAuths(byte[] user, List<byte[]> authLabels) throws IOException
{
+    assert labelsRegion != null;
+    OperationStatus[] finalOpStatus = new OperationStatus[authLabels.size()];
+    List<Mutation> puts = new ArrayList<Mutation>(authLabels.size());
+    int i = 0;
+    for (byte[] auth : authLabels) {
+      String authStr = Bytes.toString(auth);
+      int labelOrdinal = this.labelsCache.getLabelOrdinal(authStr);
+      if (labelOrdinal == 0) {
+        // This label is not yet added. 1st this should be added to the system
+        finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE,
+            new InvalidLabelException("Label '" + authStr + "' doesn't exists"));
+      } else {
+        Put p = new Put(Bytes.toBytes(labelOrdinal));
+        p.addImmutable(LABELS_TABLE_FAMILY, user, DUMMY_VALUE, LABELS_TABLE_TAGS);
+        puts.add(p);
+      }
+      i++;
+    }
+    if (mutateLabelsRegion(puts, finalOpStatus)) {
+      updateZk(false);
+    }
+    return finalOpStatus;
+  }
+
+  @Override
+  public OperationStatus[] clearAuths(byte[] user, List<byte[]> authLabels) throws
IOException {
+    assert labelsRegion != null;
+    OperationStatus[] finalOpStatus = new OperationStatus[authLabels.size()];
+    List<String> currentAuths = this.getAuths(user, true);
+    List<Mutation> deletes = new ArrayList<Mutation>(authLabels.size());
+    int i = 0;
+    for (byte[] authLabel : authLabels) {
+      String authLabelStr = Bytes.toString(authLabel);
+      if (currentAuths.contains(authLabelStr)) {
+        int labelOrdinal = this.labelsCache.getLabelOrdinal(authLabelStr);
+        assert labelOrdinal > 0;
+        Delete d = new Delete(Bytes.toBytes(labelOrdinal));
+        d.deleteColumns(LABELS_TABLE_FAMILY, user);
+        deletes.add(d);
+      } else {
+        // This label is not set for the user.
+        finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE,
+            new InvalidLabelException("Label '" + authLabelStr + "' is not set for the user
"
+                + Bytes.toString(user)));
+      }
+      i++;
+    }
+    if (mutateLabelsRegion(deletes, finalOpStatus)) {
+      updateZk(false);
+    }
+    return finalOpStatus;
+  }
+
+  /**
+   * Adds the mutations to labels region and set the results to the finalOpStatus. finalOpStatus
+   * might have some entries in it where the OpStatus is FAILURE. We will leave those and
set in
+   * others in the order.
+   * @param mutations
+   * @param finalOpStatus
+   * @return whether we need a ZK update or not.
+   */
+  private boolean mutateLabelsRegion(List<Mutation> mutations, OperationStatus[] finalOpStatus)
+      throws IOException {
+    OperationStatus[] opStatus = this.labelsRegion.batchMutate(mutations
+        .toArray(new Mutation[mutations.size()]));
+    int i = 0;
+    boolean updateZk = false;
+    for (OperationStatus status : opStatus) {
+      // Update the zk when atleast one of the mutation was added successfully.
+      updateZk = updateZk || (status.getOperationStatusCode() == OperationStatusCode.SUCCESS);
+      for (; i < finalOpStatus.length; i++) {
+        if (finalOpStatus[i] == null) {
+          finalOpStatus[i] = status;
+          break;
+        }
+      }
+    }
+    return updateZk;
+  }
+
+  @Override
+  public List<String> getAuths(byte[] user, boolean systemCall) throws IOException
{
+    assert (labelsRegion != null || systemCall);
+    if (systemCall || labelsRegion == null) {
+      return this.labelsCache.getAuths(Bytes.toString(user));
+    }
+    Scan s = new Scan();
+    s.addColumn(LABELS_TABLE_FAMILY, user);
+    Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion,
+        new Authorizations(SYSTEM_LABEL));
+    s.setFilter(filter);
+    List<String> auths = new ArrayList<String>();
+    RegionScanner scanner = this.labelsRegion.getScanner(s);
+    List<Cell> results = new ArrayList<Cell>(1);
+    while (true) {
+      scanner.next(results);
+      if (results.isEmpty()) break;
+      Cell cell = results.get(0);
+      int ordinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
+      String label = this.labelsCache.getLabel(ordinal);
+      if (label != null) {
+        auths.add(label);
+      }
+      results.clear();
+    }
+    return auths;
+  }
+
+  @Override
+  public List<Tag> createVisibilityExpTags(String visExpression, boolean withSerializationFormat,
+      boolean checkAuths) throws IOException {
+    ExpressionNode node = null;
+    try {
+      node = this.expressionParser.parse(visExpression);
+    } catch (ParseException e) {
+      throw new IOException(e);
+    }
+    node = this.expressionExpander.expand(node);
+    List<Tag> tags = new ArrayList<Tag>();
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    DataOutputStream dos = new DataOutputStream(baos);
+    List<Integer> labelOrdinals = new ArrayList<Integer>();
+    // We will be adding this tag before the visibility tags and the presence of this
+    // tag indicates we are supporting deletes with cell visibility
+    if (withSerializationFormat) {
+      tags.add(VisibilityUtils.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG);
+    }
+    Set<Integer> auths = null;
+    if (checkAuths) {
+      auths = this.labelsCache.getAuthsAsOrdinals(VisibilityUtils.getActiveUser().getShortName());
+    }
+    if (node.isSingleNode()) {
+      getLabelOrdinals(node, labelOrdinals, auths, checkAuths);
+      writeLabelOrdinalsToStream(labelOrdinals, dos);
+      tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
+      baos.reset();
+    } else {
+      NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node;
+      if (nlNode.getOperator() == Operator.OR) {
+        for (ExpressionNode child : nlNode.getChildExps()) {
+          getLabelOrdinals(child, labelOrdinals, auths, checkAuths);
+          writeLabelOrdinalsToStream(labelOrdinals, dos);
+          tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
+          baos.reset();
+          labelOrdinals.clear();
+        }
+      } else {
+        getLabelOrdinals(nlNode, labelOrdinals, auths, checkAuths);
+        writeLabelOrdinalsToStream(labelOrdinals, dos);
+        tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
+        baos.reset();
+      }
+    }
+    return tags;
+  }
+
+  protected void getLabelOrdinals(ExpressionNode node, List<Integer> labelOrdinals,
+      Set<Integer> auths, boolean checkAuths) throws IOException, InvalidLabelException
{
+    if (node.isSingleNode()) {
+      String identifier = null;
+      int labelOrdinal = 0;
+      if (node instanceof LeafExpressionNode) {
+        identifier = ((LeafExpressionNode) node).getIdentifier();
+        if (LOG.isTraceEnabled()) {
+          LOG.trace("The identifier is " + identifier);
+        }
+        labelOrdinal = this.labelsCache.getLabelOrdinal(identifier);
+        checkAuths(auths, labelOrdinal, identifier, checkAuths);
+      } else {
+        // This is a NOT node.
+        LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node)
+            .getChildExps().get(0);
+        identifier = lNode.getIdentifier();
+        labelOrdinal = this.labelsCache.getLabelOrdinal(identifier);
+        checkAuths(auths, labelOrdinal, identifier, checkAuths);
+        labelOrdinal = -1 * labelOrdinal; // Store NOT node as -ve ordinal.
+      }
+      if (labelOrdinal == 0) {
+        throw new InvalidLabelException("Invalid visibility label " + identifier);
+      }
+      labelOrdinals.add(labelOrdinal);
+    } else {
+      List<ExpressionNode> childExps = ((NonLeafExpressionNode) node).getChildExps();
+      for (ExpressionNode child : childExps) {
+        getLabelOrdinals(child, labelOrdinals, auths, checkAuths);
+      }
+    }
+  }
+
+  private void checkAuths(Set<Integer> auths, int labelOrdinal, String identifier,
+      boolean checkAuths) throws IOException {
+    if (checkAuths) {
+      if (auths == null || (!auths.contains(labelOrdinal))) {
+        throw new AccessDeniedException("Visibility label " + identifier
+            + " not authorized for the user " + VisibilityUtils.getActiveUser().getShortName());
+      }
+    }
+  }
+
+  /**
+   * This will sort the passed labels in ascending oder and then will write one after the
other
+   * to the passed stream.
+   * @param labelOrdinals Unsorted label ordinals
+   * @param dos Stream where to write the labels.
+   * @throws IOException When IOE during writes to Stream.
+   */
+  protected void writeLabelOrdinalsToStream(List<Integer> labelOrdinals, DataOutputStream
dos)
+      throws IOException {
+    Collections.sort(labelOrdinals);
+    for (Integer labelOrdinal : labelOrdinals) {
+      StreamUtils.writeRawVInt32(dos, labelOrdinal);
+    }
+  }
+
+  protected void updateZk(boolean labelAddition) throws IOException {
+    // We will add to zookeeper here.
+    // TODO we should add the delta only to zk. Else this will be a very heavy op and when
there are
+    // so many labels and auth in the system, we will end up adding lots of data to zk. Most
+    // possibly we will exceed zk node data limit!
+    Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths
=
+        extractLabelsAndAuths(getExistingLabelsWithAuths());
+    Map<String, Integer> existingLabels = labelsAndUserAuths.getFirst();
+    Map<String, List<Integer>> userAuths = labelsAndUserAuths.getSecond();
+    if (labelAddition) {
+      byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(existingLabels);
+      this.labelsCache.writeToZookeeper(serialized, true);
+    } else {
+      byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths);
+      this.labelsCache.writeToZookeeper(serialized, false);
+    }
+  }
+
+  @Override
+  public VisibilityExpEvaluator getVisibilityExpEvaluator(Authorizations authorizations)
+      throws IOException {
+    // If a super user issues a get/scan, he should be able to scan the cells
+    // irrespective of the Visibility labels
+    if (isReadFromSuperUser()) {
+      return new VisibilityExpEvaluator() {
+        @Override
+        public boolean evaluate(Cell cell) throws IOException {
+          return true;
+        }
+      };
+    }
+    List<String> authLabels = null;
+    for (ScanLabelGenerator scanLabelGenerator : scanLabelGenerators) {
+      try {
+        // null authorizations to be handled inside SLG impl.
+        authLabels = scanLabelGenerator.getLabels(VisibilityUtils.getActiveUser(), authorizations);
+        authLabels = (authLabels == null) ? new ArrayList<String>() : authLabels;
+        authorizations = new Authorizations(authLabels);
+      } catch (Throwable t) {
+        LOG.error(t);
+        throw new IOException(t);
+      }
+    }
+    int labelsCount = this.labelsCache.getLabelsCount();
+    final BitSet bs = new BitSet(labelsCount + 1); // ordinal is index 1 based
+    if (authLabels != null) {
+      for (String authLabel : authLabels) {
+        int labelOrdinal = this.labelsCache.getLabelOrdinal(authLabel);
+        if (labelOrdinal != 0) {
+          bs.set(labelOrdinal);
+        }
+      }
+    }
+
+    return new VisibilityExpEvaluator() {
+      @Override
+      public boolean evaluate(Cell cell) throws IOException {
+        boolean visibilityTagPresent = false;
+        // Save an object allocation where we can
+        if (cell.getTagsLengthUnsigned() > 0) {
+          Iterator<Tag> tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
+              cell.getTagsLengthUnsigned());
+          while (tagsItr.hasNext()) {
+            boolean includeKV = true;
+            Tag tag = tagsItr.next();
+            if (tag.getType() == VISIBILITY_TAG_TYPE) {
+              visibilityTagPresent = true;
+              int offset = tag.getTagOffset();
+              int endOffset = offset + tag.getTagLength();
+              while (offset < endOffset) {
+                Pair<Integer, Integer> result = StreamUtils
+                    .readRawVarint32(tag.getBuffer(), offset);
+                int currLabelOrdinal = result.getFirst();
+                if (currLabelOrdinal < 0) {
+                  // check for the absence of this label in the Scan Auth labels
+                  // ie. to check BitSet corresponding bit is 0
+                  int temp = -currLabelOrdinal;
+                  if (bs.get(temp)) {
+                    includeKV = false;
+                    break;
+                  }
+                } else {
+                  if (!bs.get(currLabelOrdinal)) {
+                    includeKV = false;
+                    break;
+                  }
+                }
+                offset += result.getSecond();
+              }
+              if (includeKV) {
+                // We got one visibility expression getting evaluated to true. Good to include
this
+                // KV in the result then.
+                return true;
+              }
+            }
+          }
+        }
+        return !(visibilityTagPresent);
+      }
+    };
+  }
+
+  protected boolean isReadFromSuperUser() throws IOException {
+    byte[] user = Bytes.toBytes(VisibilityUtils.getActiveUser().getShortName());
+    return havingSystemAuth(user);
+  }
+
+  @Override
+  public boolean havingSystemAuth(byte[] user) throws IOException {
+    List<String> auths = this.getAuths(user, true);
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("The auths for user " + Bytes.toString(user) + " are " + auths);
+    }
+    return auths.contains(SYSTEM_LABEL);
+  }
+
+  @Override
+  public boolean matchVisibility(List<Tag> putVisTags, Byte putTagsFormat, List<Tag>
deleteVisTags,
+      Byte deleteTagsFormat) throws IOException {
+    if ((deleteTagsFormat != null && deleteTagsFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT)
+        && (putTagsFormat == null || putTagsFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT))
{
+      if (putVisTags.size() == 0) {
+        // Early out if there are no tags in the cell
+        return false;
+      }
+      if (putTagsFormat == null) {
+        return matchUnSortedVisibilityTags(putVisTags, deleteVisTags);
+      } else {
+        return matchOrdinalSortedVisibilityTags(putVisTags, deleteVisTags);
+      }
+    }
+    throw new IOException("Unexpected tag format passed for comparison, deleteTagsFormat
: "
+        + deleteTagsFormat + ", putTagsFormat : " + putTagsFormat);
+  }
+
+  /**
+   * @param putVisTags Visibility tags in Put Mutation
+   * @param deleteVisTags Visibility tags in Delete Mutation
+   * @return true when all the visibility tags in Put matches with visibility tags in Delete.
+   * This is used when, at least one set of tags are not sorted based on the label ordinal.
+   */
+  private static boolean matchUnSortedVisibilityTags(List<Tag> putVisTags,
+      List<Tag> deleteVisTags) throws IOException {
+    return compareTagsOrdinals(sortTagsBasedOnOrdinal(putVisTags),
+        sortTagsBasedOnOrdinal(deleteVisTags));
+  }
+
+  /**
+   * @param putVisTags Visibility tags in Put Mutation
+   * @param deleteVisTags Visibility tags in Delete Mutation
+   * @return true when all the visibility tags in Put matches with visibility tags in Delete.
+   * This is used when both the set of tags are sorted based on the label ordinal.
+   */
+  private static boolean matchOrdinalSortedVisibilityTags(List<Tag> putVisTags,
+      List<Tag> deleteVisTags) {
+    boolean matchFound = false;
+    // If the size does not match. Definitely we are not comparing the equal tags.
+    if ((deleteVisTags.size()) == putVisTags.size()) {
+      for (Tag tag : deleteVisTags) {
+        matchFound = false;
+        for (Tag givenTag : putVisTags) {
+          if (Bytes.equals(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength(),
+              givenTag.getBuffer(), givenTag.getTagOffset(), givenTag.getTagLength())) {
+            matchFound = true;
+            break;
+          }
+        }
+        if (!matchFound) break;
+      }
+    }
+    return matchFound;
+  }
+
+  private static List<List<Integer>> sortTagsBasedOnOrdinal(List<Tag> tags)
throws IOException {
+    List<List<Integer>> fullTagsList = new ArrayList<List<Integer>>();
+    for (Tag tag : tags) {
+      if (tag.getType() == VISIBILITY_TAG_TYPE) {
+        getSortedTagOrdinals(fullTagsList, tag);
+      }
+    }
+    return fullTagsList;
+  }
+
+  private static void getSortedTagOrdinals(List<List<Integer>> fullTagsList,
Tag tag)
+      throws IOException {
+    List<Integer> tagsOrdinalInSortedOrder = new ArrayList<Integer>();
+    int offset = tag.getTagOffset();
+    int endOffset = offset + tag.getTagLength();
+    while (offset < endOffset) {
+      Pair<Integer, Integer> result = StreamUtils.readRawVarint32(tag.getBuffer(),
offset);
+      tagsOrdinalInSortedOrder.add(result.getFirst());
+      offset += result.getSecond();
+    }
+    Collections.sort(tagsOrdinalInSortedOrder);
+    fullTagsList.add(tagsOrdinalInSortedOrder);
+  }
+
+  /*
+   * @return true when all the visibility tags in Put matches with visibility tags in Delete.
+   */
+  private static boolean compareTagsOrdinals(List<List<Integer>> putVisTags,
+      List<List<Integer>> deleteVisTags) {
+    boolean matchFound = false;
+    if (deleteVisTags.size() == putVisTags.size()) {
+      for (List<Integer> deleteTagOrdinals : deleteVisTags) {
+        matchFound = false;
+        for (List<Integer> tagOrdinals : putVisTags) {
+          if (deleteTagOrdinals.equals(tagOrdinals)) {
+            matchFound = true;
+            break;
+          }
+        }
+        if (!matchFound) break;
+      }
+    }
+    return matchFound;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/d7986121/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/EnforcingScanLabelGenerator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/EnforcingScanLabelGenerator.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/EnforcingScanLabelGenerator.java
index 7d0320a..90a8ff9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/EnforcingScanLabelGenerator.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/EnforcingScanLabelGenerator.java
@@ -37,10 +37,10 @@ public class EnforcingScanLabelGenerator implements ScanLabelGenerator
{
   private static final Log LOG = LogFactory.getLog(EnforcingScanLabelGenerator.class);
 
   private Configuration conf;
-  private VisibilityLabelsManager labelsManager;
+  private VisibilityLabelsCache labelsCache;
 
   public EnforcingScanLabelGenerator() {
-    this.labelsManager = VisibilityLabelsManager.get();
+    this.labelsCache = VisibilityLabelsCache.get();
   }
 
   @Override
@@ -59,7 +59,7 @@ public class EnforcingScanLabelGenerator implements ScanLabelGenerator {
     if (authorizations != null) {
       LOG.warn("Dropping authorizations requested by user " + userName + ": " + authorizations);
     }
-    return this.labelsManager.getAuths(userName);
+    return this.labelsCache.getAuths(userName);
   }
 
 }


Mime
View raw message