incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/2] git commit: Adding new type in Blur. Read mask type will allow for data to be searched and discovered but not viewed.
Date Tue, 13 Oct 2015 13:50:13 GMT
Adding new type in Blur.  Read mask type will allow for data to be searched and discovered
but not viewed.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/0e6aed41
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/0e6aed41
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/0e6aed41

Branch: refs/heads/master
Commit: 0e6aed41bc36996f88eebc9d93d04bb25a7f79fc
Parents: 647c66b
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Oct 13 09:50:04 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Oct 13 09:50:04 2015 -0400

----------------------------------------------------------------------
 .../index/FilterAccessControlFactory.java       |  82 +++++--
 .../security/index/SecureAtomicReader.java      |  78 ++++--
 .../apache/blur/analysis/BaseFieldManager.java  |  37 ++-
 .../org/apache/blur/analysis/FieldManager.java  |   5 -
 .../blur/analysis/FieldTypeDefinition.java      |  12 +
 .../type/ReadMaskFieldTypeDefinition.java       | 157 ++++++++++++
 .../type/StringFieldTypeDefinition.java         |   2 +-
 .../type/ReadMaskFieldTypeDefinitionTest.java   | 238 +++++++++++++++++++
 8 files changed, 560 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6aed41/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java
----------------------------------------------------------------------
diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java
b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java
index 3b78cf3..dcd3a1c 100644
--- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java
+++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java
@@ -32,7 +32,9 @@ import org.apache.blur.lucene.security.search.DocumentVisibilityFilterCacheStrat
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.FloatField;
 import org.apache.lucene.document.IntField;
 import org.apache.lucene.document.LongField;
@@ -45,10 +47,12 @@ import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.IndexableFieldType;
 import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
 
 public class FilterAccessControlFactory extends AccessControlFactory {
 
@@ -318,13 +322,19 @@ public class FilterAccessControlFactory extends AccessControlFactory
{
 
     @Override
     public Iterable<IndexableField> lastStepBeforeIndexing(Iterable<IndexableField>
fields) {
+      return processFieldMasks(fields);
+    }
+
+    public static Iterable<IndexableField> processFieldMasks(Iterable<IndexableField>
fields) {
       Set<String> fieldsToMask = getFieldsToMask(fields);
       if (fieldsToMask.isEmpty()) {
         return fields;
       }
       List<IndexableField> result = new ArrayList<IndexableField>();
       for (IndexableField field : fields) {
-        if (fieldsToMask.contains(field.name())) {
+        IndexableFieldType fieldType = field.fieldType();
+        // If field is to be indexed and is to be read masked.
+        if (fieldsToMask.contains(field.name()) && fieldType.indexed()) {
           // If field is a doc value, then don't bother indexing.
           if (!isDocValue(field)) {
             if (isStoredField(field)) {
@@ -348,12 +358,22 @@ public class FilterAccessControlFactory extends AccessControlFactory
{
       Set<String> result = new HashSet<String>();
       for (IndexableField field : fields) {
         if (field.name().equals(READ_MASK_FIELD)) {
-          result.add(field.stringValue());
+          result.add(getFieldNameOnly(field.stringValue()));
         }
       }
       return result;
     }
 
+    private static String getFieldNameOnly(String s) {
+      // remove any stored messages
+      int indexOf = s.indexOf('|');
+      if (indexOf < 0) {
+        return s;
+      } else {
+        return s.substring(0, indexOf);
+      }
+    }
+
     private static boolean isStoredField(IndexableField field) {
       if (field instanceof StoredField) {
         return true;
@@ -362,47 +382,71 @@ public class FilterAccessControlFactory extends AccessControlFactory
{
     }
 
     private static IndexableField createMaskField(IndexableField field) {
+      FieldType fieldTypeNotStored = getFieldTypeNotStored(field);
+      String name = field.name() + READ_MASK_SUFFIX;
       if (field instanceof DoubleField) {
         DoubleField f = (DoubleField) field;
-        return new DoubleField(field.name() + READ_MASK_SUFFIX, (double) f.numericValue(),
f.fieldType());
+        return new DoubleField(name, (double) f.numericValue(), fieldTypeNotStored);
       } else if (field instanceof FloatField) {
         FloatField f = (FloatField) field;
-        return new FloatField(field.name() + READ_MASK_SUFFIX, (float) f.numericValue(),
f.fieldType());
+        return new FloatField(name, (float) f.numericValue(), fieldTypeNotStored);
       } else if (field instanceof IntField) {
         IntField f = (IntField) field;
-        return new IntField(field.name() + READ_MASK_SUFFIX, (int) f.numericValue(), f.fieldType());
+        return new IntField(name, (int) f.numericValue(), fieldTypeNotStored);
       } else if (field instanceof LongField) {
         LongField f = (LongField) field;
-        return new LongField(field.name() + READ_MASK_SUFFIX, (long) f.numericValue(), f.fieldType());
+        return new LongField(name, (long) f.numericValue(), fieldTypeNotStored);
       } else if (field instanceof StringField) {
         StringField f = (StringField) field;
-        if (f.fieldType() == StringField.TYPE_NOT_STORED) {
-          return new StringField(field.name() + READ_MASK_SUFFIX, f.stringValue(), Store.NO);
-        } else {
-          return new StringField(field.name() + READ_MASK_SUFFIX, f.stringValue(), Store.YES);
-        }
+        return new StringField(name, f.stringValue(), Store.NO);
       } else if (field instanceof StringField) {
         TextField f = (TextField) field;
         Reader readerValue = f.readerValue();
         if (readerValue != null) {
-          return new TextField(field.name() + READ_MASK_SUFFIX, readerValue);
+          return new TextField(name, readerValue);
         }
         TokenStream tokenStreamValue = f.tokenStreamValue();
         if (tokenStreamValue != null) {
-          return new TextField(field.name() + READ_MASK_SUFFIX, tokenStreamValue);
+          return new TextField(name, tokenStreamValue);
         }
-        Store s;
-        if (f.fieldType() == StringField.TYPE_NOT_STORED) {
-          s = Store.NO;
-        } else {
-          s = Store.YES;
+        return new TextField(name, f.stringValue(), Store.NO);
+      } else if (field.getClass().equals(Field.class)) {
+        Field f = (Field) field;
+        String stringValue = f.stringValue();
+        if (stringValue != null) {
+          return new Field(name, stringValue, fieldTypeNotStored);
+        }
+        BytesRef binaryValue = f.binaryValue();
+        if (binaryValue != null) {
+          return new Field(name, binaryValue, fieldTypeNotStored);
+        }
+        Number numericValue = f.numericValue();
+        if (numericValue != null) {
+          throw new RuntimeException("Field [" + field + "] with type [" + field.getClass()
+ "] is not supported.");
         }
-        return new TextField(field.name() + READ_MASK_SUFFIX, f.stringValue(), s);
+        Reader readerValue = f.readerValue();
+        if (readerValue != null) {
+          return new Field(name, readerValue, fieldTypeNotStored);
+        }
+        TokenStream tokenStreamValue = f.tokenStreamValue();
+        if (tokenStreamValue != null) {
+          return new Field(name, tokenStreamValue, fieldTypeNotStored);
+        }
+        throw new RuntimeException("Field [" + field + "] with type [" + field.getClass()
+ "] is not supported.");
       } else {
         throw new RuntimeException("Field [" + field + "] with type [" + field.getClass()
+ "] is not supported.");
       }
     }
 
+    private static FieldType getFieldTypeNotStored(IndexableField indexableField) {
+      Field field = (Field) indexableField;
+      FieldType fieldType = field.fieldType();
+      FieldType result = new FieldType(fieldType);
+      result.setStored(false);
+      result.freeze();
+      return result;
+    }
+
     private static boolean isDocValue(IndexableField field) {
       if (field instanceof BinaryDocValuesField) {
         return true;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6aed41/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java
----------------------------------------------------------------------
diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java
b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java
index b81727b..2a17edb 100644
--- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java
+++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java
@@ -18,7 +18,9 @@ package org.apache.blur.lucene.security.index;
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.HashSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.lucene.index.AtomicReader;
@@ -39,6 +41,8 @@ import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
 
+import com.google.common.base.Splitter;
+
 /**
  * The current {@link SecureAtomicReader} will protect access to documents based
  * on the {@link AccessControl} object.
@@ -112,7 +116,7 @@ public class SecureAtomicReader extends FilterAtomicReader {
     if (_accessControl.hasAccess(ReadType.DOCUMENT_FETCH_READ, docID)) {
       GetReadMaskFields getReadMaskFields = new GetReadMaskFields();
       in.document(docID, getReadMaskFields);
-      Set<String> readMaskFields = getReadMaskFields.getReadMaskFields();
+      Map<String, String> readMaskFields = getReadMaskFields.getReadMaskFields();
       if (readMaskFields.isEmpty()) {
         in.document(docID, visitor);
       } else {
@@ -169,56 +173,78 @@ public class SecureAtomicReader extends FilterAtomicReader {
   private static class ReadMaskStoredFieldVisitor extends StoredFieldVisitor {
 
     private final StoredFieldVisitor _visitor;
-    private final Set<String> _readMaskFields;
+    private final Map<String, String> _readMaskFieldsAndMessages;
 
-    public ReadMaskStoredFieldVisitor(StoredFieldVisitor visitor, Set<String> readMaskFields)
{
+    public ReadMaskStoredFieldVisitor(StoredFieldVisitor visitor, Map<String, String>
readMaskFieldsAndMessages) {
       _visitor = visitor;
-      _readMaskFields = readMaskFields;
+      _readMaskFieldsAndMessages = readMaskFieldsAndMessages;
     }
 
     @Override
     public Status needsField(FieldInfo fieldInfo) throws IOException {
-      if (_readMaskFields.contains(fieldInfo.name)) {
-        return Status.NO;
-      }
       return Status.YES;
     }
 
     @Override
     public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
-      _visitor.binaryField(fieldInfo, value);
+      if (!checkReadMask(fieldInfo)) {
+        _visitor.binaryField(fieldInfo, value);
+      }
+    }
+
+    private boolean checkReadMask(FieldInfo fieldInfo) throws IOException {
+      final String message = _readMaskFieldsAndMessages.get(fieldInfo.name);
+      if (message != null) {
+        if (message.isEmpty()) {
+          return true;
+        }
+        _visitor.stringField(fieldInfo, message);
+        return true;
+      }
+      return false;
     }
 
     @Override
     public void stringField(FieldInfo fieldInfo, String value) throws IOException {
-      _visitor.stringField(fieldInfo, value);
+      if (!checkReadMask(fieldInfo)) {
+        _visitor.stringField(fieldInfo, value);
+      }
     }
 
     @Override
     public void intField(FieldInfo fieldInfo, int value) throws IOException {
-      _visitor.intField(fieldInfo, value);
+      if (!checkReadMask(fieldInfo)) {
+        _visitor.intField(fieldInfo, value);
+      }
     }
 
     @Override
     public void longField(FieldInfo fieldInfo, long value) throws IOException {
-      _visitor.longField(fieldInfo, value);
+      if (!checkReadMask(fieldInfo)) {
+        _visitor.longField(fieldInfo, value);
+      }
     }
 
     @Override
     public void floatField(FieldInfo fieldInfo, float value) throws IOException {
-      _visitor.floatField(fieldInfo, value);
+      if (!checkReadMask(fieldInfo)) {
+        _visitor.floatField(fieldInfo, value);
+      }
     }
 
     @Override
     public void doubleField(FieldInfo fieldInfo, double value) throws IOException {
-      _visitor.doubleField(fieldInfo, value);
+      if (!checkReadMask(fieldInfo)) {
+        _visitor.doubleField(fieldInfo, value);
+      }
     }
 
   }
 
   private static class GetReadMaskFields extends StoredFieldVisitor {
 
-    private Set<String> _fields = new HashSet<String>();
+    private Map<String, String> _fieldsAndMessages = new HashMap<String, String>();
+    private Splitter splitter = Splitter.on('|');
 
     @Override
     public Status needsField(FieldInfo fieldInfo) throws IOException {
@@ -230,11 +256,27 @@ public class SecureAtomicReader extends FilterAtomicReader {
 
     @Override
     public void stringField(FieldInfo fieldInfo, String value) throws IOException {
-      _fields.add(value);
+      Iterable<String> split = splitter.split(value);
+      Iterator<String> iterator = split.iterator();
+      String field;
+      String message = null;
+      if (iterator.hasNext()) {
+        field = iterator.next();
+      } else {
+        return;
+      }
+      if (iterator.hasNext()) {
+        message = iterator.next();
+      }
+      if (message != null) {
+        _fieldsAndMessages.put(field, message);
+      } else {
+        _fieldsAndMessages.put(field, "");
+      }
     }
 
-    Set<String> getReadMaskFields() {
-      return _fields;
+    Map<String, String> getReadMaskFields() {
+      return _fieldsAndMessages;
     }
 
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6aed41/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
index 42f33c9..c4ed8c8 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/BaseFieldManager.java
@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -37,6 +38,7 @@ import org.apache.blur.analysis.type.FloatFieldTypeDefinition;
 import org.apache.blur.analysis.type.IntFieldTypeDefinition;
 import org.apache.blur.analysis.type.LongFieldTypeDefinition;
 import org.apache.blur.analysis.type.NumericFieldTypeDefinition;
+import org.apache.blur.analysis.type.ReadMaskFieldTypeDefinition;
 import org.apache.blur.analysis.type.StoredFieldTypeDefinition;
 import org.apache.blur.analysis.type.StringFieldTypeDefinition;
 import org.apache.blur.analysis.type.TextFieldTypeDefinition;
@@ -58,7 +60,6 @@ import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SortField;
 
@@ -67,6 +68,12 @@ public abstract class BaseFieldManager extends FieldManager {
   private static final Log LOG = LogFactory.getLog(BaseFieldManager.class);
   private static final Map<String, String> EMPTY_MAP = new HashMap<String, String>();
   private static final boolean DEFAULT_MULTI_VALUE_FIELD_VALUE = true;
+  private static final Comparator<FieldTypeDefinition> POST_PROCESSING_COMPARATOR =
new Comparator<FieldTypeDefinition>() {
+    @Override
+    public int compare(FieldTypeDefinition o1, FieldTypeDefinition o2) {
+      return Integer.compare(o2.getPostProcessingPriority(), o1.getPostProcessingPriority());
+    }
+  };
 
   private final ConcurrentMap<String, Set<String>> _columnToSubColumn = new ConcurrentHashMap<String,
Set<String>>();
   private final ConcurrentMap<String, FieldTypeDefinition> _fieldNameToDefMap = new
ConcurrentHashMap<String, FieldTypeDefinition>();
@@ -120,6 +127,7 @@ public abstract class BaseFieldManager extends FieldManager {
     registerType(SpatialRecursivePrefixTreeStrategyFieldTypeDefinition.class);
     registerType(AclReadFieldTypeDefinition.class);
     registerType(AclDiscoverFieldTypeDefinition.class);
+    registerType(ReadMaskFieldTypeDefinition.class);
     _fieldLessField = fieldLessField;
     _strict = strict;
     _defaultMissingFieldLessIndexing = defaultMissingFieldLessIndexing;
@@ -227,6 +235,7 @@ public abstract class BaseFieldManager extends FieldManager {
     addDefaultFields(fields, rowId, record);
     addFieldExistance(fields, record);
     Map<String, Integer> fieldCounts = new HashMap<String, Integer>();
+    List<FieldTypeDefinition> postProcessingFieldTypes = new ArrayList<FieldTypeDefinition>();
     for (Column column : columns) {
       String name = column.getName();
       String value = column.getValue();
@@ -243,6 +252,9 @@ public abstract class BaseFieldManager extends FieldManager {
             false, DEFAULT_MULTI_VALUE_FIELD_VALUE, getDefaultMissingFieldProps());
         fieldTypeDefinition = getFieldTypeDefinition(family, column);
       }
+      if (fieldTypeDefinition.isPostProcessingSupported()) {
+        postProcessingFieldTypes.add(fieldTypeDefinition);
+      }
       String fieldName = fieldTypeDefinition.getFieldName();
       Integer count = fieldCounts.get(fieldName);
       if (count == null) {
@@ -267,15 +279,24 @@ public abstract class BaseFieldManager extends FieldManager {
         }
       }
     }
-    return fields;
-  }
-
-  public List<IndexableField> getIndexableFields(String fieldname, String fieldValue)
throws IOException {
-    throw new RuntimeException("Not implemented.");
+    if (!postProcessingFieldTypes.isEmpty()) {
+      Collections.sort(postProcessingFieldTypes, POST_PROCESSING_COMPARATOR);
+      Iterable<? extends Field> iterable = fields;
+      for (FieldTypeDefinition fieldTypeDefinition : postProcessingFieldTypes) {
+        iterable = fieldTypeDefinition.executePostProcessing(iterable);
+      }
+      return toList(iterable);
+    } else {
+      return fields;
+    }
   }
 
-  public int getFieldId(String fieldName) throws IOException {
-    throw new RuntimeException("Not implemented.");
+  private List<Field> toList(Iterable<? extends Field> iterable) {
+    List<Field> fields = new ArrayList<Field>();
+    for (Field field : iterable) {
+      fields.add(field);
+    }
+    return fields;
   }
 
   private void addFieldExistance(List<Field> fields, Record record) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6aed41/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
index 6dd70b2..a30bd9d 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldManager.java
@@ -24,7 +24,6 @@ import java.util.Set;
 import org.apache.blur.thrift.generated.Record;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SortField;
 
@@ -40,10 +39,6 @@ public abstract class FieldManager {
    */
   public abstract List<Field> getFields(String rowId, Record record) throws IOException;
 
-  public abstract List<IndexableField> getIndexableFields(String fieldname, String
fieldValue) throws IOException;
-  
-  public abstract int getFieldId(String fieldName) throws IOException;
-
   /**
    * Adds a column definition.
    * 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6aed41/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java b/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java
index 0cae0d6..c000426 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/FieldTypeDefinition.java
@@ -282,4 +282,16 @@ public abstract class FieldTypeDefinition {
    */
   public abstract String readTerm(BytesRef byteRef);
 
+  public boolean isPostProcessingSupported() {
+    return false;
+  }
+
+  public int getPostProcessingPriority() {
+    return 0;
+  }
+
+  public Iterable<? extends Field> executePostProcessing(Iterable<? extends Field>
fields) {
+    throw new RuntimeException("Not Implemented.");
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6aed41/blur-query/src/main/java/org/apache/blur/analysis/type/ReadMaskFieldTypeDefinition.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/ReadMaskFieldTypeDefinition.java
b/blur-query/src/main/java/org/apache/blur/analysis/type/ReadMaskFieldTypeDefinition.java
new file mode 100644
index 0000000..cba8c57
--- /dev/null
+++ b/blur-query/src/main/java/org/apache/blur/analysis/type/ReadMaskFieldTypeDefinition.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.analysis.type;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.blur.analysis.FieldTypeDefinition;
+import org.apache.blur.lucene.security.index.FilterAccessControlFactory.FilterAccessControlWriter;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BytesRef;
+
+public class ReadMaskFieldTypeDefinition extends FieldTypeDefinition {
+
+  private static final String INTERNAL_FIELDNAME = "_readmask_";
+  private static final KeywordAnalyzer KEYWORD_ANALYZER = new KeywordAnalyzer();
+  private static final String READ_MASK = "read-mask";
+  private static final Collection<String> ALT_FIELD_NAMES;
+
+  static {
+    ALT_FIELD_NAMES = new HashSet<String>();
+    ALT_FIELD_NAMES.add(INTERNAL_FIELDNAME);
+  }
+
+  @Override
+  public String getName() {
+    return READ_MASK;
+  }
+
+  @Override
+  public void configure(String fieldNameForThisInstance, Map<String, String> properties,
Configuration configuration) {
+
+  }
+
+  @Override
+  public Collection<String> getAlternateFieldNames() {
+    return ALT_FIELD_NAMES;
+  }
+
+  @Override
+  public boolean isAlternateFieldNamesSharedAcrossInstances() {
+    return true;
+  }
+
+  @Override
+  public Iterable<? extends Field> getFieldsForColumn(String family, Column column)
{
+    String name = getName(family, column.getName());
+    List<Field> fields = new ArrayList<Field>();
+    fields.add(new StoredField(name, column.getValue()));
+    fields.add(new StringField(INTERNAL_FIELDNAME, column.getValue(), Store.YES));
+    return fields;
+  }
+
+  @Override
+  public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column,
String subName) {
+    return makeIterable(new StringField(INTERNAL_FIELDNAME, column.getValue(), Store.YES));
+  }
+
+  @Override
+  public boolean isPostProcessingSupported() {
+    return true;
+  }
+
+  @Override
+  public int getPostProcessingPriority() {
+    return Integer.MAX_VALUE;
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public Iterable<? extends Field> executePostProcessing(Iterable<? extends Field>
fields) {
+    Iterable<IndexableField> doc = FilterAccessControlWriter.processFieldMasks((Iterable<IndexableField>)
fields);
+    return (Iterable<? extends Field>) doc;
+  }
+
+  @Override
+  public Analyzer getAnalyzerForIndex(String fieldName) {
+    // shouldn't be used ever
+    return KEYWORD_ANALYZER;
+  }
+
+  @Override
+  public Analyzer getAnalyzerForQuery(String fieldName) {
+    return KEYWORD_ANALYZER;
+  }
+
+  @Override
+  public boolean checkSupportForFuzzyQuery() {
+    return true;
+  }
+
+  @Override
+  public boolean checkSupportForWildcardQuery() {
+    return true;
+  }
+
+  @Override
+  public boolean checkSupportForPrefixQuery() {
+    return true;
+  }
+
+  @Override
+  public boolean checkSupportForRegexQuery() {
+    return true;
+  }
+
+  @Override
+  public boolean isNumeric() {
+    return false;
+  }
+
+  @Override
+  public boolean checkSupportForCustomQuery() {
+    return false;
+  }
+
+  @Override
+  public boolean checkSupportForSorting() {
+    return false;
+  }
+
+  @Override
+  public String readTerm(BytesRef byteRef) {
+    return byteRef.utf8ToString();
+  }
+
+  @Override
+  public SortField getSortField(boolean reverse) {
+    throw new RuntimeException("Sort not supported.");
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6aed41/blur-query/src/main/java/org/apache/blur/analysis/type/StringFieldTypeDefinition.java
----------------------------------------------------------------------
diff --git a/blur-query/src/main/java/org/apache/blur/analysis/type/StringFieldTypeDefinition.java
b/blur-query/src/main/java/org/apache/blur/analysis/type/StringFieldTypeDefinition.java
index eecf7e1..bc80631 100644
--- a/blur-query/src/main/java/org/apache/blur/analysis/type/StringFieldTypeDefinition.java
+++ b/blur-query/src/main/java/org/apache/blur/analysis/type/StringFieldTypeDefinition.java
@@ -129,6 +129,6 @@ public class StringFieldTypeDefinition extends FieldTypeDefinition {
 
   @Override
   public String readTerm(BytesRef byteRef) {
-	return byteRef.utf8ToString();
+    return byteRef.utf8ToString();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e6aed41/blur-query/src/test/java/org/apache/blur/analysis/type/ReadMaskFieldTypeDefinitionTest.java
----------------------------------------------------------------------
diff --git a/blur-query/src/test/java/org/apache/blur/analysis/type/ReadMaskFieldTypeDefinitionTest.java
b/blur-query/src/test/java/org/apache/blur/analysis/type/ReadMaskFieldTypeDefinitionTest.java
new file mode 100644
index 0000000..5d69c7d
--- /dev/null
+++ b/blur-query/src/test/java/org/apache/blur/analysis/type/ReadMaskFieldTypeDefinitionTest.java
@@ -0,0 +1,238 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.analysis.type;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.blur.analysis.BaseFieldManager;
+import org.apache.blur.analysis.FieldTypeDefinition;
+import org.apache.blur.analysis.NoStopWordStandardAnalyzer;
+import org.apache.blur.lucene.search.SuperParser;
+import org.apache.blur.lucene.security.index.AccessControlFactory;
+import org.apache.blur.lucene.security.index.FilterAccessControlFactory;
+import org.apache.blur.lucene.security.search.SecureIndexSearcher;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.ScoreType;
+import org.apache.blur.utils.BlurConstants;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Version;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ReadMaskFieldTypeDefinitionTest {
+  private static final String FAM = "fam";
+  private static final String FAM2 = "fam2";
+
+  private Directory _dir = new RAMDirectory();
+  private AccessControlFactory _accessControlFactory = new FilterAccessControlFactory();
+
+  private BaseFieldManager _fieldManager;
+
+  @Before
+  public void setup() throws IOException {
+    _fieldManager = getFieldManager(new NoStopWordStandardAnalyzer());
+    setupFieldManager(_fieldManager);
+
+    List<List<Field>> docs = new ArrayList<List<Field>>();
+    {
+      Record record = new Record();
+      record.setFamily(FAM);
+      record.setRecordId("1234");
+      record.addToColumns(new Column("string", "value"));
+      record.addToColumns(new Column("read", "a&b"));
+      record.addToColumns(new Column("string2", "value should not read"));
+      record.addToColumns(new Column("mask", "fam.string2|READ_MASK"));
+      List<Field> fields = _fieldManager.getFields("1234", record);
+      fields.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE,
Store.NO));
+      docs.add(debug(fields));
+    }
+    {
+      Record record = new Record();
+      record.setFamily(FAM);
+      record.setRecordId("5678");
+      record.addToColumns(new Column("string", "value"));
+      record.addToColumns(new Column("read", "a&c"));
+      record.addToColumns(new Column("mask", "fam.string"));
+      docs.add(debug(_fieldManager.getFields("1234", record)));
+    }
+
+    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, _fieldManager.getAnalyzerForIndex());
+    IndexWriter writer = new IndexWriter(_dir, conf);
+    writer.addDocuments(docs);
+    writer.close();
+  }
+
+  private List<Field> debug(List<Field> fields) {
+    // System.out.println("----Document");
+    // for (Field field : fields) {
+    // System.out.println(field);
+    // }
+    return fields;
+  }
+
+  @Test
+  public void test1RowQuery() throws IOException, ParseException {
+    test(0, true, null);
+  }
+
+  @Test
+  public void test1RecordQuery() throws IOException, ParseException {
+    test(0, false, null);
+  }
+
+  @Test
+  public void test2RowQuery() throws IOException, ParseException {
+    test(1, true, Arrays.asList("a", "b"));
+  }
+
+  @Test
+  public void test2RecordQuery() throws IOException, ParseException {
+    test(1, false, Arrays.asList("a", "b"));
+  }
+
+  @Test
+  public void test3RowQuery() throws IOException, ParseException {
+    test(1, true, Arrays.asList("a", "b", "c"));
+  }
+
+  @Test
+  public void test3RecordQuery() throws IOException, ParseException {
+    test(2, false, Arrays.asList("a", "b", "c"));
+  }
+
+  @Test
+  public void test4RowQuery() throws IOException, ParseException {
+    test(0, true, Arrays.asList("a"));
+  }
+
+  @Test
+  public void test4RecordQuery() throws IOException, ParseException {
+    test(0, false, Arrays.asList("a"));
+  }
+
+  private AccessControlFactory getAccessControlFactory() {
+    return _accessControlFactory;
+  }
+
+  private void setupFieldManager(BaseFieldManager fieldManager) throws IOException {
+    fieldManager.addColumnDefinition(FAM, "string", null, false, "string", false, false,
null);
+    fieldManager.addColumnDefinition(FAM, "string2", null, false, "string", false, false,
null);
+    fieldManager.addColumnDefinition(FAM, "read", null, false, "acl-read", false, false,
null);
+    fieldManager.addColumnDefinition(FAM, "mask", null, false, "read-mask", false, false,
null);
+    fieldManager.addColumnDefinition(FAM2, "string", null, false, "string", false, false,
null);
+    fieldManager.addColumnDefinition(FAM2, "read", null, false, "acl-read", false, false,
null);
+  }
+
+  protected BaseFieldManager getFieldManager(Analyzer a) throws IOException {
+    BaseFieldManager fieldManager = new BaseFieldManager(BlurConstants.SUPER, a, new Configuration())
{
+      @Override
+      protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName)
{
+        return true;
+      }
+
+      @Override
+      protected void tryToLoad(String fieldName) {
+
+      }
+
+      @Override
+      protected List<String> getFieldNamesToLoad() throws IOException {
+        return new ArrayList<String>();
+      }
+    };
+    return fieldManager;
+  }
+
+  private void test(int expected, boolean rowQuery, Collection<String> readAuthorizations)
throws IOException,
+      ParseException {
+    DirectoryReader reader = DirectoryReader.open(_dir);
+    SuperParser parser = new SuperParser(Version.LUCENE_43, _fieldManager, rowQuery, null,
ScoreType.SUPER, new Term(
+        BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE));
+
+    Query query = parser.parse("fam.string:value");
+
+    Collection<String> discoverAuthorizations = null;
+    Set<String> discoverableFields = null;
+    IndexSearcher searcher = new SecureIndexSearcher(reader, getAccessControlFactory(), readAuthorizations,
+        discoverAuthorizations, discoverableFields);
+
+    checkTerms(searcher, "fam.string2");
+
+    TopDocs topDocs = searcher.search(query, 10);
+    assertEquals(expected, topDocs.totalHits);
+
+    for (int hit = 0; hit < topDocs.totalHits; hit++) {
+      int doc = topDocs.scoreDocs[hit].doc;
+      Document document = searcher.doc(doc);
+      String recordId = document.get("recordid");
+      if (recordId.equals("1234")) {
+        String s = document.get("fam.string2");
+        assertEquals("READ_MASK", s);
+      } else if (recordId.equals("5678")) {
+        String s = document.get("fam.string");
+        assertNull(s);
+      }
+    }
+
+    reader.close();
+  }
+
+  private void checkTerms(IndexSearcher searcher, String fieldName) throws IOException {
+    IndexReader reader = searcher.getIndexReader();
+    for (AtomicReaderContext context : reader.leaves()) {
+      AtomicReader atomicReader = context.reader();
+      Fields fields = atomicReader.fields();
+      Terms terms = fields.terms(fieldName);
+      TermsEnum iterator = terms.iterator(null);
+      BytesRef bytesRef = iterator.next();
+      if (bytesRef != null) {
+        System.out.println(bytesRef.utf8ToString());
+        fail("There are only restricted terms for this field [" + fieldName + "]");
+      }
+    }
+  }
+}


Mime
View raw message