incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Adding read mask to blur document security.
Date Tue, 06 Oct 2015 13:39:18 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master b736c0a45 -> 9ba62e6c9


Adding read mask to blur document security.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/9ba62e6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/9ba62e6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/9ba62e6c

Branch: refs/heads/master
Commit: 9ba62e6c91f4325774b6b40d0e1d8cc5f02306c4
Parents: b736c0a
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Oct 6 09:39:09 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Oct 6 09:39:09 2015 -0400

----------------------------------------------------------------------
 .../document/DocumentVisiblityField.java        |   3 +-
 .../security/index/AccessControlFactory.java    |   4 +
 .../security/index/AccessControlReader.java     |   4 +-
 .../security/index/AccessControlWriter.java     |  27 ++-
 .../index/FilterAccessControlFactory.java       | 133 ++++++++++++++
 .../blur/lucene/security/index/ReadType.java    |   2 +-
 .../security/index/SecureAtomicReader.java      | 180 ++++++++++++++++++-
 .../index/FilterSecureAtomicReaderTest.java     |   9 +
 .../index/SecureAtomicReaderTestBase.java       | 123 +++++++++----
 9 files changed, 438 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/document/DocumentVisiblityField.java
----------------------------------------------------------------------
diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/document/DocumentVisiblityField.java
b/blur-document-security/src/main/java/org/apache/blur/lucene/security/document/DocumentVisiblityField.java
index d3fd4f8..09a098f 100644
--- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/document/DocumentVisiblityField.java
+++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/document/DocumentVisiblityField.java
@@ -25,9 +25,8 @@ import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.IndexableField;
 
-public class DocumentVisiblityField extends Field implements IndexableField {
+public class DocumentVisiblityField extends Field {
 
   public static final FieldType TYPE_NOT_STORED = new FieldType();
   public static final FieldType TYPE_STORED = new FieldType();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java
----------------------------------------------------------------------
diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java
b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java
index 25ad35f..40dd486 100644
--- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java
+++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java
@@ -24,6 +24,10 @@ public abstract class AccessControlFactory {
   public abstract String getDiscoverFieldName();
   
   public abstract String getReadFieldName();
+  
+  public abstract String getReadMaskFieldName();
+  
+  public abstract String getReadMaskFieldSuffix();
 
   public abstract AccessControlWriter getWriter();
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java
----------------------------------------------------------------------
diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java
b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java
index 51b554a..8ea214d 100644
--- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java
+++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java
@@ -25,11 +25,13 @@ public abstract class AccessControlReader implements Cloneable {
 
   public final boolean hasAccess(ReadType type, int docID) throws IOException {
     switch (type) {
-    case DOCS_ENUM:
+    
     case LIVEDOCS:
       return readOrDiscoverAccess(docID);
     case DOCUMENT_FETCH_DISCOVER:
       return discoverAccess(docID);
+    case DOCS_ENUM:
+    case TERMS_ENUM:
     case BINARY_DOC_VALUE:
     case DOCUMENT_FETCH_READ:
     case NORM_VALUE:

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlWriter.java
----------------------------------------------------------------------
diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlWriter.java
b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlWriter.java
index 035cde0..e351745 100644
--- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlWriter.java
+++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlWriter.java
@@ -24,16 +24,38 @@ import org.apache.lucene.index.IndexableField;
 
 public abstract class AccessControlWriter {
 
+  /**
+   * Adds document read visibility rule to document.
+   */
   public abstract Iterable<IndexableField> addReadVisiblity(String read, Iterable<IndexableField>
fields);
 
+  /**
+   * Adds document discover visibility rule to document.
+   */
   public abstract Iterable<IndexableField> addDiscoverVisiblity(String discover, Iterable<IndexableField>
fields);
 
+  /**
+   * Adds a read mask to document. If a field has been masked the value can not
+   * be viewed, but if a search utilizes the tokens from the field the document
+   * can be found.
+   */
+  public abstract Iterable<IndexableField> addReadMask(String fieldToMask, Iterable<IndexableField>
fields);
+
+  /**
+   * This method should be called as the document is being added to the index
+   * writer.
+   * 
+   * @param fields
+   * @return
+   */
+  public abstract Iterable<IndexableField> lastStepBeforeIndexing(Iterable<IndexableField>
fields);
+
   protected Iterable<IndexableField> addField(Iterable<IndexableField> fields,
IndexableField... fieldsToAdd) {
     if (fields instanceof Document) {
       Document document = (Document) fields;
       if (fieldsToAdd != null) {
         for (IndexableField field : fieldsToAdd) {
-          document.add(field);    
+          document.add(field);
         }
       }
       return document;
@@ -44,9 +66,10 @@ public abstract class AccessControlWriter {
     }
     if (fieldsToAdd != null) {
       for (IndexableField field : fieldsToAdd) {
-        list.add(field);    
+        list.add(field);
       }
     }
     return list;
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java
----------------------------------------------------------------------
diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java
b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java
index 038b0c7..3b78cf3 100644
--- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java
+++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java
@@ -17,7 +17,11 @@
 package org.apache.blur.lucene.security.index;
 
 import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import org.apache.blur.lucene.security.DocumentAuthorizations;
@@ -25,7 +29,19 @@ import org.apache.blur.lucene.security.document.DocumentVisiblityField;
 import org.apache.blur.lucene.security.search.BitSetDocumentVisibilityFilterCacheStrategy;
 import org.apache.blur.lucene.security.search.DocumentVisibilityFilter;
 import org.apache.blur.lucene.security.search.DocumentVisibilityFilterCacheStrategy;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.DoubleField;
 import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexableField;
@@ -38,6 +54,8 @@ public class FilterAccessControlFactory extends AccessControlFactory {
 
   public static final String DISCOVER_FIELD = "_discover_";
   public static final String READ_FIELD = "_read_";
+  public static final String READ_MASK_FIELD = "_readmask_";
+  public static final String READ_MASK_SUFFIX = "$" + READ_MASK_FIELD;
 
   @Override
   public String getDiscoverFieldName() {
@@ -50,6 +68,16 @@ public class FilterAccessControlFactory extends AccessControlFactory {
   }
 
   @Override
+  public String getReadMaskFieldName() {
+    return READ_MASK_FIELD;
+  }
+
+  @Override
+  public String getReadMaskFieldSuffix() {
+    return READ_MASK_SUFFIX;
+  }
+
+  @Override
   public AccessControlWriter getWriter() {
     return new FilterAccessControlWriter();
   }
@@ -283,6 +311,111 @@ public class FilterAccessControlFactory extends AccessControlFactory
{
       return addField(fields, new DocumentVisiblityField(DISCOVER_FIELD, discover, Store.YES));
     }
 
+    @Override
+    public Iterable<IndexableField> addReadMask(String fieldToMask, Iterable<IndexableField>
fields) {
+      return addField(fields, new StoredField(READ_MASK_FIELD, fieldToMask));
+    }
+
+    @Override
+    public Iterable<IndexableField> lastStepBeforeIndexing(Iterable<IndexableField>
fields) {
+      Set<String> fieldsToMask = getFieldsToMask(fields);
+      if (fieldsToMask.isEmpty()) {
+        return fields;
+      }
+      List<IndexableField> result = new ArrayList<IndexableField>();
+      for (IndexableField field : fields) {
+        if (fieldsToMask.contains(field.name())) {
+          // If field is a doc value, then don't bother indexing.
+          if (!isDocValue(field)) {
+            if (isStoredField(field)) {
+              // Stored fields are not indexed, and the document fetch check
+              // handles the mask.
+              result.add(field);
+            } else {
+              IndexableField mask = createMaskField(field);
+              result.add(field);
+              result.add(mask);
+            }
+          }
+        } else {
+          result.add(field);
+        }
+      }
+      return result;
+    }
+
+    private static Set<String> getFieldsToMask(Iterable<IndexableField> fields)
{
+      Set<String> result = new HashSet<String>();
+      for (IndexableField field : fields) {
+        if (field.name().equals(READ_MASK_FIELD)) {
+          result.add(field.stringValue());
+        }
+      }
+      return result;
+    }
+
+    private static boolean isStoredField(IndexableField field) {
+      if (field instanceof StoredField) {
+        return true;
+      }
+      return false;
+    }
+
+    private static IndexableField createMaskField(IndexableField field) {
+      if (field instanceof DoubleField) {
+        DoubleField f = (DoubleField) field;
+        return new DoubleField(field.name() + READ_MASK_SUFFIX, (double) f.numericValue(),
f.fieldType());
+      } else if (field instanceof FloatField) {
+        FloatField f = (FloatField) field;
+        return new FloatField(field.name() + READ_MASK_SUFFIX, (float) f.numericValue(),
f.fieldType());
+      } else if (field instanceof IntField) {
+        IntField f = (IntField) field;
+        return new IntField(field.name() + READ_MASK_SUFFIX, (int) f.numericValue(), f.fieldType());
+      } else if (field instanceof LongField) {
+        LongField f = (LongField) field;
+        return new LongField(field.name() + READ_MASK_SUFFIX, (long) f.numericValue(), f.fieldType());
+      } else if (field instanceof StringField) {
+        StringField f = (StringField) field;
+        if (f.fieldType() == StringField.TYPE_NOT_STORED) {
+          return new StringField(field.name() + READ_MASK_SUFFIX, f.stringValue(), Store.NO);
+        } else {
+          return new StringField(field.name() + READ_MASK_SUFFIX, f.stringValue(), Store.YES);
+        }
+      } else if (field instanceof StringField) {
+        TextField f = (TextField) field;
+        Reader readerValue = f.readerValue();
+        if (readerValue != null) {
+          return new TextField(field.name() + READ_MASK_SUFFIX, readerValue);
+        }
+        TokenStream tokenStreamValue = f.tokenStreamValue();
+        if (tokenStreamValue != null) {
+          return new TextField(field.name() + READ_MASK_SUFFIX, tokenStreamValue);
+        }
+        Store s;
+        if (f.fieldType() == StringField.TYPE_NOT_STORED) {
+          s = Store.NO;
+        } else {
+          s = Store.YES;
+        }
+        return new TextField(field.name() + READ_MASK_SUFFIX, f.stringValue(), s);
+      } else {
+        throw new RuntimeException("Field [" + field + "] with type [" + field.getClass()
+ "] is not supported.");
+      }
+    }
+
+    private static boolean isDocValue(IndexableField field) {
+      if (field instanceof BinaryDocValuesField) {
+        return true;
+      } else if (field instanceof NumericDocValuesField) {
+        return true;
+      } else if (field instanceof SortedDocValuesField) {
+        return true;
+      } else if (field instanceof SortedSetDocValuesField) {
+        return true;
+      } else {
+        return false;
+      }
+    }
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/ReadType.java
----------------------------------------------------------------------
diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/ReadType.java
b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/ReadType.java
index dec08b0..dcfbe2c 100644
--- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/ReadType.java
+++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/ReadType.java
@@ -17,5 +17,5 @@
 package org.apache.blur.lucene.security.index;
 
 public enum ReadType {
-  LIVEDOCS, DOCUMENT_FETCH_READ, DOCUMENT_FETCH_DISCOVER, NUMERIC_DOC_VALUE, BINARY_DOC_VALUE,
SORTED_DOC_VALUE, NORM_VALUE, SORTED_SET_DOC_VALUE, DOCS_ENUM, QUERY
+  LIVEDOCS, DOCUMENT_FETCH_READ, DOCUMENT_FETCH_DISCOVER, NUMERIC_DOC_VALUE, BINARY_DOC_VALUE,
SORTED_DOC_VALUE, NORM_VALUE, SORTED_SET_DOC_VALUE, DOCS_ENUM, QUERY, TERMS_ENUM
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java
----------------------------------------------------------------------
diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java
b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java
index 9155769..b81727b 100644
--- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java
+++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java
@@ -18,6 +18,7 @@ package org.apache.blur.lucene.security.index;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Set;
 
 import org.apache.lucene.index.AtomicReader;
@@ -33,6 +34,7 @@ import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
@@ -108,11 +110,17 @@ public class SecureAtomicReader extends FilterAtomicReader {
   @Override
   public void document(int docID, final StoredFieldVisitor visitor) throws IOException {
     if (_accessControl.hasAccess(ReadType.DOCUMENT_FETCH_READ, docID)) {
-      in.document(docID, visitor);
+      GetReadMaskFields getReadMaskFields = new GetReadMaskFields();
+      in.document(docID, getReadMaskFields);
+      Set<String> readMaskFields = getReadMaskFields.getReadMaskFields();
+      if (readMaskFields.isEmpty()) {
+        in.document(docID, visitor);
+      } else {
+        in.document(docID, new ReadMaskStoredFieldVisitor(visitor, readMaskFields));
+      }
       return;
     }
     if (_accessControl.hasAccess(ReadType.DOCUMENT_FETCH_DISCOVER, docID)) {
-      // TODO add way to perform code when visitor runs here....
       in.document(docID, new StoredFieldVisitor() {
         @Override
         public Status needsField(FieldInfo fieldInfo) throws IOException {
@@ -158,6 +166,79 @@ public class SecureAtomicReader extends FilterAtomicReader {
     }
   }
 
+  private static class ReadMaskStoredFieldVisitor extends StoredFieldVisitor {
+
+    private final StoredFieldVisitor _visitor;
+    private final Set<String> _readMaskFields;
+
+    public ReadMaskStoredFieldVisitor(StoredFieldVisitor visitor, Set<String> readMaskFields)
{
+      _visitor = visitor;
+      _readMaskFields = readMaskFields;
+    }
+
+    @Override
+    public Status needsField(FieldInfo fieldInfo) throws IOException {
+      if (_readMaskFields.contains(fieldInfo.name)) {
+        return Status.NO;
+      }
+      return Status.YES;
+    }
+
+    @Override
+    public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
+      _visitor.binaryField(fieldInfo, value);
+    }
+
+    @Override
+    public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+      _visitor.stringField(fieldInfo, value);
+    }
+
+    @Override
+    public void intField(FieldInfo fieldInfo, int value) throws IOException {
+      _visitor.intField(fieldInfo, value);
+    }
+
+    @Override
+    public void longField(FieldInfo fieldInfo, long value) throws IOException {
+      _visitor.longField(fieldInfo, value);
+    }
+
+    @Override
+    public void floatField(FieldInfo fieldInfo, float value) throws IOException {
+      _visitor.floatField(fieldInfo, value);
+    }
+
+    @Override
+    public void doubleField(FieldInfo fieldInfo, double value) throws IOException {
+      _visitor.doubleField(fieldInfo, value);
+    }
+
+  }
+
+  private static class GetReadMaskFields extends StoredFieldVisitor {
+
+    private Set<String> _fields = new HashSet<String>();
+
+    @Override
+    public Status needsField(FieldInfo fieldInfo) throws IOException {
+      if (fieldInfo.name.equals(FilterAccessControlFactory.READ_MASK_FIELD)) {
+        return Status.YES;
+      }
+      return Status.NO;
+    }
+
+    @Override
+    public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+      _fields.add(value);
+    }
+
+    Set<String> getReadMaskFields() {
+      return _fields;
+    }
+
+  }
+
   @Override
   public Fields fields() throws IOException {
     return new SecureFields(in.fields(), _accessControl, maxDoc());
@@ -315,7 +396,40 @@ public class SecureAtomicReader extends FilterAtomicReader {
       if (terms == null) {
         return null;
       }
-      return new SecureTerms(terms, _accessControlReader, _maxDoc);
+      Terms readMask = getReadMaskTerms(in, field);
+      SecureTerms secureTerms = new SecureTerms(terms, _accessControlReader, _maxDoc);
+      if (readMask == null) {
+        return secureTerms;
+      } else {
+        return new ReadMaskTerms(secureTerms, readMask);
+      }
+    }
+
+    private Terms getReadMaskTerms(Fields in, String field) throws IOException {
+      return in.terms(field + FilterAccessControlFactory.READ_MASK_SUFFIX);
+    }
+
+  }
+
+  static class ReadMaskTerms extends FilterTerms {
+
+    private final Terms _readMask;
+
+    public ReadMaskTerms(Terms in, Terms readMask) {
+      super(in);
+      _readMask = readMask;
+    }
+
+    @Override
+    public TermsEnum iterator(TermsEnum reuse) throws IOException {
+      TermsEnum maskTermsEnum = _readMask.iterator(null);
+      return new ReadMaskTermsEnum(maskTermsEnum, in.iterator(reuse));
+    }
+
+    @Override
+    public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException
{
+      TermsEnum maskTermsEnum = _readMask.intersect(compiled, startTerm);
+      return new ReadMaskTermsEnum(maskTermsEnum, in.intersect(compiled, startTerm));
     }
 
   }
@@ -342,6 +456,44 @@ public class SecureAtomicReader extends FilterAtomicReader {
     }
   }
 
+  static class ReadMaskTermsEnum extends FilterTermsEnum {
+
+    private final TermsEnum _maskTermsEnum;
+
+    public ReadMaskTermsEnum(TermsEnum maskTermsEnum, TermsEnum realTermsEnum) {
+      super(realTermsEnum);
+      _maskTermsEnum = maskTermsEnum;
+    }
+
+    @Override
+    public BytesRef next() throws IOException {
+      while (true) {
+        BytesRef ref = in.next();
+        if (ref == null) {
+          return null;
+        }
+        if (!_maskTermsEnum.seekExact(ref, true)) {
+          return ref;
+        }
+        if (checkDocs()) {
+          return ref;
+        }
+      }
+    }
+
+    private boolean checkDocs() throws IOException {
+      DocsEnum maskDocsEnum = _maskTermsEnum.docs(null, null, DocsEnum.FLAG_NONE);
+      DocsEnum docsEnum = in.docs(null, null, DocsEnum.FLAG_NONE);
+      int docId;
+      while ((docId = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+        if (maskDocsEnum.advance(docId) != docId) {
+          return true;
+        }
+      }
+      return false;
+    }
+  }
+
   static class SecureTermsEnum extends FilterTermsEnum {
 
     private final int _maxDoc;
@@ -354,6 +506,28 @@ public class SecureAtomicReader extends FilterAtomicReader {
     }
 
     @Override
+    public BytesRef next() throws IOException {
+      BytesRef t;
+      while ((t = in.next()) != null) {
+        if (hasAccess(t)) {
+          return t;
+        }
+      }
+      return null;
+    }
+
+    private boolean hasAccess(BytesRef term) throws IOException {
+      DocsEnum docsEnum = in.docs(null, null, DocsEnum.FLAG_NONE);
+      int docId;
+      while ((docId = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+        if (_accessControlReader.hasAccess(ReadType.TERMS_ENUM, docId)) {
+          return true;
+        }
+      }
+      return false;
+    }
+
+    @Override
     public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
       Bits secureLiveDocs = getSecureLiveDocs(liveDocs, _maxDoc, _accessControlReader);
       return in.docs(secureLiveDocs, reuse, flags);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/FilterSecureAtomicReaderTest.java
----------------------------------------------------------------------
diff --git a/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/FilterSecureAtomicReaderTest.java
b/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/FilterSecureAtomicReaderTest.java
index 748d749..75a0476 100644
--- a/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/FilterSecureAtomicReaderTest.java
+++ b/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/FilterSecureAtomicReaderTest.java
@@ -16,8 +16,12 @@
  */
 package org.apache.blur.lucene.security.index;
 
+import java.io.IOException;
+
 import org.apache.blur.lucene.security.index.AccessControlFactory;
 import org.apache.blur.lucene.security.index.FilterAccessControlFactory;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.junit.Test;
 
 public class FilterSecureAtomicReaderTest extends SecureAtomicReaderTestBase {
 
@@ -27,5 +31,10 @@ public class FilterSecureAtomicReaderTest extends SecureAtomicReaderTestBase
{
   public AccessControlFactory getAccessControlFactory() {
     return _accessControlFactory;
   }
+  
+  @Test
+  public void testTermWalk() throws IOException, ParseException {
+    super.testTermWalk();
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java
----------------------------------------------------------------------
diff --git a/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java
b/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java
index f87171a..fc40f42 100644
--- a/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java
+++ b/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java
@@ -28,10 +28,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.blur.lucene.security.index.AccessControlFactory;
-import org.apache.blur.lucene.security.index.AccessControlReader;
-import org.apache.blur.lucene.security.index.AccessControlWriter;
-import org.apache.blur.lucene.security.index.SecureAtomicReader;
+import org.apache.blur.lucene.security.search.SecureIndexSearcher;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
@@ -44,7 +41,6 @@ import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocsEnum;
 import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -56,8 +52,6 @@ import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
@@ -69,17 +63,23 @@ import org.junit.Test;
 
 public abstract class SecureAtomicReaderTestBase {
 
+  private Set<String> discoverableFields = new HashSet<String>(Arrays.asList("info"));
+  private List<String> readAuthorizations = Arrays.asList("r1");
+  private List<String> discoverAuthorizations = Arrays.asList("d1");
+
   public abstract AccessControlFactory getAccessControlFactory();
 
   @Test
   public void testLiveDocs() throws IOException {
     SecureAtomicReader secureReader = getSecureReader();
     Bits liveDocs = secureReader.getLiveDocs();
-    assertEquals(4, liveDocs.length());
+    assertEquals(6, liveDocs.length());
     assertTrue(liveDocs.get(0));
     assertTrue(liveDocs.get(1));
     assertTrue(liveDocs.get(2));
     assertFalse(liveDocs.get(3));
+    assertTrue(liveDocs.get(4));
+    assertTrue(liveDocs.get(5));
     secureReader.close();
   }
 
@@ -232,30 +232,45 @@ public abstract class SecureAtomicReaderTestBase {
   public void testTermWalk() throws IOException, ParseException {
     SecureAtomicReader secureReader = getSecureReader();
     Fields fields = secureReader.fields();
-    for (String field : fields) {
-      Terms terms = fields.terms(field);
-      TermsEnum termsEnum = terms.iterator(null);
-      BytesRef ref;
-      while ((ref = termsEnum.next()) != null) {
-        System.out.println(field + " " + ref.utf8ToString());
-        DocsEnum docsEnum = termsEnum.docs(null, null);
-        int doc;
-        while ((doc = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
-          System.out.println(field + " " + ref.utf8ToString() + " " + doc);
-        }
-      }
-    }
+    // for (String field : fields) {
+    // Terms terms = fields.terms(field);
+    // TermsEnum termsEnum = terms.iterator(null);
+    // BytesRef ref;
+    // while ((ref = termsEnum.next()) != null) {
+    // System.out.println(field + " " + ref.utf8ToString());
+    // DocsEnum docsEnum = termsEnum.docs(null, null);
+    // int doc;
+    // while ((doc = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+    // System.out.println(field + " " + ref.utf8ToString() + " " + doc);
+    // }
+    // }
+    // }
+
+    assertEquals(0, getTermCount(fields, "termmask")); //read mask
+    assertEquals(0, getTermCount(fields, "shouldnotsee")); //discover
+    assertEquals(1, getTermCount(fields, "test"));
+
     secureReader.close();
   }
 
+  private int getTermCount(Fields fields, String field) throws IOException {
+    Terms terms = fields.terms(field);
+    TermsEnum termsEnum = terms.iterator(null);
+    int count = 0;
+    while (termsEnum.next() != null) {
+      count++;
+    }
+    return count;
+  }
+
   @Test
   public void testQuery() throws IOException, ParseException {
-    SecureAtomicReader secureReader = getSecureReader();
+    SecureIndexSearcher searcher = getSecureIndexSearcher();
     QueryParser parser = new QueryParser(Version.LUCENE_43, "nothing", new KeywordAnalyzer());
     Query query = parser.parse("test:test");
-    IndexSearcher searcher = new IndexSearcher(secureReader);
+
     TopDocs topDocs = searcher.search(query, 10);
-    assertEquals(3, topDocs.totalHits);
+    assertEquals(5, topDocs.totalHits);
     {
       int doc = topDocs.scoreDocs[0].doc;
       assertEquals(0, doc);
@@ -277,20 +292,42 @@ public abstract class SecureAtomicReaderTestBase {
       assertEquals("test", document.get("test"));
       assertEquals("info", document.get("info"));
     }
+    {
+      int doc = topDocs.scoreDocs[3].doc;
+      assertEquals(4, doc);
+      Document document = searcher.doc(doc);
+      assertNull(document.get("test"));
+      assertEquals("info", document.get("info"));
+    }
+    {
+      int doc = topDocs.scoreDocs[4].doc;
+      assertEquals(5, doc);
+      Document document = searcher.doc(doc);
+      assertEquals("test", document.get("test"));
+      assertEquals("info", document.get("info"));
+    }
+  }
 
-    secureReader.close();
+  private SecureIndexSearcher getSecureIndexSearcher() throws IOException {
+    DirectoryReader reader = createReader();
+    return new SecureIndexSearcher(reader, getAccessControlFactory(), Arrays.asList("r1"),
Arrays.asList("d1"),
+        discoverableFields);
   }
 
   private SecureAtomicReader getSecureReader() throws IOException {
-    AtomicReader baseReader = createReader();
-    Set<String> dicoverableFields = new HashSet<String>();
-    dicoverableFields.add("info");
-    AccessControlReader accessControlReader = getAccessControlFactory().getReader(Arrays.asList("r1"),
-        Arrays.asList("d1"), dicoverableFields);
+    AtomicReader baseReader = createAtomicReader();
+    AccessControlReader accessControlReader = getAccessControlFactory().getReader(readAuthorizations,
+        discoverAuthorizations, discoverableFields);
     return new SecureAtomicReader(baseReader, accessControlReader);
   }
 
-  private AtomicReader createReader() throws IOException {
+  private AtomicReader createAtomicReader() throws IOException {
+    DirectoryReader reader = createReader();
+    List<AtomicReaderContext> leaves = reader.leaves();
+    return leaves.get(0).reader();
+  }
+
+  private DirectoryReader createReader() throws IOException {
     IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
     Directory dir = new RAMDirectory();
     IndexWriter writer = new IndexWriter(dir, conf);
@@ -299,17 +336,24 @@ public abstract class SecureAtomicReaderTestBase {
     addDoc(writer, accessControlWriter, "r2", "d1", 1);
     addDoc(writer, accessControlWriter, "r1", "d2", 2);
     addDoc(writer, accessControlWriter, "r2", "d2", 3);
+    addDoc(writer, accessControlWriter, "r1", "d1", 4, "test");
+    addDoc(writer, accessControlWriter, "r1", "d1", 5, "termmask");
     writer.close();
 
-    DirectoryReader reader = DirectoryReader.open(dir);
-    List<AtomicReaderContext> leaves = reader.leaves();
-    return leaves.get(0).reader();
+    return DirectoryReader.open(dir);
   }
 
-  private void addDoc(IndexWriter writer, AccessControlWriter accessControlWriter, String
read, String discover, int doc)
-      throws IOException {
-    writer.addDocument(accessControlWriter.addDiscoverVisiblity(discover,
-        accessControlWriter.addReadVisiblity(read, getDoc(doc))));
+  private void addDoc(IndexWriter writer, AccessControlWriter accessControlWriter, String
read, String discover,
+      int doc, String... readMaskFields) throws IOException {
+    Iterable<IndexableField> fields = getDoc(doc);
+    fields = accessControlWriter.addReadVisiblity(read, fields);
+    fields = accessControlWriter.addDiscoverVisiblity(discover, fields);
+    if (readMaskFields != null) {
+      for (String readMaskField : readMaskFields) {
+        fields = accessControlWriter.addReadMask(readMaskField, fields);
+      }
+    }
+    writer.addDocument(accessControlWriter.lastStepBeforeIndexing(fields));
   }
 
   private Iterable<IndexableField> getDoc(int i) {
@@ -319,6 +363,9 @@ public abstract class SecureAtomicReaderTestBase {
     if (i == 3) {
       document.add(new StringField("shouldnotsee", "shouldnotsee", Store.YES));
     }
+    if (i == 5) {
+      document.add(new StringField("termmask", "term", Store.YES));
+    }
     document.add(new NumericDocValuesField("number", i));
     document.add(new BinaryDocValuesField("bin", new BytesRef(Integer.toString(i).getBytes())));
     document.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(i).getBytes())));


Mime
View raw message