manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1570837 - in /manifoldcf/trunk: connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ framework/agents/src/main...
Date Sat, 22 Feb 2014 13:22:34 GMT
Author: kwright
Date: Sat Feb 22 13:22:33 2014
New Revision: 1570837

URL: http://svn.apache.org/r1570837
Log:
Redefine RepositoryDocument security in a way that should be better compatible with future
additions

Modified:
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
    manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1570837&r1=1570836&r2=1570837&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
Sat Feb 22 13:22:33 2014
@@ -25,6 +25,7 @@ import java.io.InputStream;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Iterator;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.http.conn.ClientConnectionManager;
@@ -404,6 +405,29 @@ public class ElasticSearchConnector exte
     return null;
   }
 
+  /** Convert an unqualified ACL to qualified form.
+  * @param acl is the initial, unqualified ACL.
+  * @param authorityNameString is the name of the governing authority for this document's
acls, or null if none.
+  * @param activities is the activities object, so we can report what's happening.
+  * @return the modified ACL.
+  */
+  protected static String[] convertACL(String[] acl, String authorityNameString, IOutputAddActivity
activities)
+    throws ManifoldCFException
+  {
+    if (acl != null)
+    {
+      String[] rval = new String[acl.length];
+      int i = 0;
+      while (i < rval.length)
+      {
+        rval[i] = activities.qualifyAccessToken(authorityNameString,acl[i]);
+        i++;
+      }
+      return rval;
+    }
+    return new String[0];
+  }
+
   @Override
   public int addOrReplaceDocument(String documentURI, String outputDescription,
       RepositoryDocument document, String authorityNameString,
@@ -413,11 +437,40 @@ public class ElasticSearchConnector exte
     HttpClient client = getSession();
     ElasticSearchConfig config = getConfigParameters(null);
     InputStream inputStream = document.getBinaryStream();
+    // For ES, we have to have fixed fields only; nothing else is possible b/c we don't have
+    // default field values.
+    String[] acls = null;
+    String[] denyAcls = null;
+    String[] shareAcls = null;
+    String[] shareDenyAcls = null;
+    Iterator<String> a = document.securityTypesIterator();
+    while (a.hasNext())
+    {
+      String securityType = a.next();
+      String[] convertedAcls = convertACL(document.getSecurityACL(securityType),authorityNameString,activities);
+      String[] convertedDenyAcls = convertACL(document.getSecurityDenyACL(securityType),authorityNameString,activities);
+      if (securityType.equals(RepositoryDocument.SECURITY_TYPE_DOCUMENT))
+      {
+        acls = convertedAcls;
+        denyAcls = convertedDenyAcls;
+      }
+      else if (securityType.equals(RepositoryDocument.SECURITY_TYPE_SHARE))
+      {
+        shareAcls = convertedAcls;
+        shareDenyAcls = convertedDenyAcls;
+      }
+      else
+      {
+        // Don't know how to deal with it
+        return DOCUMENTSTATUS_REJECTED;
+      }
+    }
+    
     long startTime = System.currentTimeMillis();
     ElasticSearchIndex oi = new ElasticSearchIndex(client, config);
     try
     {
-      oi.execute(documentURI, document, inputStream);
+      oi.execute(documentURI, document, inputStream, acls, denyAcls, shareAcls, shareDenyAcls);
       if (oi.getResult() != Result.OK)
         return DOCUMENTSTATUS_REJECTED;
       return DOCUMENTSTATUS_ACCEPTED;

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1570837&r1=1570836&r2=1570837&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
Sat Feb 22 13:22:33 2014
@@ -57,14 +57,23 @@ public class ElasticSearchIndex extends 
   private class IndexRequestEntity implements HttpEntity
   {
 
-    private RepositoryDocument document;
-    private InputStream inputStream;
+    private final RepositoryDocument document;
+    private final InputStream inputStream;
+    private final String[] acls;
+    private final String[] denyAcls;
+    private final String[] shareAcls;
+    private final String[] shareDenyAcls;
 
-    public IndexRequestEntity(RepositoryDocument document, InputStream inputStream)
+    public IndexRequestEntity(RepositoryDocument document, InputStream inputStream,
+      String[] acls, String[] denyAcls, String[] shareAcls, String[] shareDenyAcls)
       throws ManifoldCFException
     {
       this.document = document;
       this.inputStream = inputStream;
+      this.acls = acls;
+      this.denyAcls = denyAcls;
+      this.shareAcls = shareAcls;
+      this.shareDenyAcls = shareDenyAcls;
     }
 
     @Override
@@ -109,16 +118,8 @@ public class ElasticSearchIndex extends 
           needComma = writeField(pw, needComma, fieldName, fieldValues);
         }
 
-        needComma = writeACLs(pw, needComma, "document", document.getACL(), document.getDenyACL());
-        needComma = writeACLs(pw, needComma, "share", document.getShareACL(), document.getShareDenyACL());
-
-        int directoryCount = document.countDirectoryACLs();
-        int q = 0;
-        while (q < directoryCount)
-        {
-          needComma = writeACLs(pw, needComma, "directory_" + q, document.getDirectoryACL(q),
document.getDirectoryDenyACL(q));
-          q++;
-        }
+        needComma = writeACLs(pw, needComma, "document", acls, denyAcls);
+        needComma = writeACLs(pw, needComma, "share", shareAcls, shareDenyAcls);
 
         if(inputStream!=null){
           if(needComma){
@@ -254,7 +255,8 @@ public class ElasticSearchIndex extends 
   *@return false to indicate that the document was rejected.
   */
   public boolean execute(String documentURI, RepositoryDocument document, 
-      InputStream inputStream) throws ManifoldCFException, ServiceInterruption
+    InputStream inputStream, String[] acls, String[] denyAcls, String[] shareAcls, String[]
shareDenyAcls)
+    throws ManifoldCFException, ServiceInterruption
   {
     String idField;
     try
@@ -268,7 +270,7 @@ public class ElasticSearchIndex extends 
 
     StringBuffer url = getApiUrl(config.getIndexType() + "/" + idField, false);
     HttpPut put = new HttpPut(url.toString());
-    put.setEntity(new IndexRequestEntity(document, inputStream));
+    put.setEntity(new IndexRequestEntity(document, inputStream, acls, denyAcls, shareAcls,
shareDenyAcls));
     if (call(put) == false)
       return false;
     if ("true".equals(checkJson(jsonStatus)))

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1570837&r1=1570836&r2=1570837&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Sat Feb 22 13:22:33 2014
@@ -518,29 +518,29 @@ public class HttpPoster
     if (maxDocumentLength != null && document.getBinaryLength() > maxDocumentLength.longValue())
       return false;
     
-    // Convert the incoming acls to qualified forms
-    String[] shareAcls = convertACL(document.getShareACL(),authorityNameString,activities);
-    String[] shareDenyAcls = convertACL(document.getShareDenyACL(),authorityNameString,activities);
-    String[] acls = convertACL(document.getACL(),authorityNameString,activities);
-    String[] denyAcls = convertACL(document.getDenyACL(),authorityNameString,activities);
-
-    Map<Integer,String[]> directoryAclsMap = new HashMap<Integer,String[]>();
-    Map<Integer,String[]> directoryDenyAclsMap = new HashMap<Integer,String[]>();
-    int directoryCount = document.countDirectoryACLs();
-    int q = 0;
-    while (q < directoryCount)
-    {
-      String[] directoryAcls = convertACL(document.getDirectoryACL(q),authorityNameString,activities);
-      String[] directoryDenyAcls = convertACL(document.getDirectoryDenyACL(q),authorityNameString,activities);
-      directoryAclsMap.put(Integer.valueOf(q), directoryAcls);
-      directoryDenyAclsMap.put(Integer.valueOf(q), directoryDenyAcls);
-      q++;
+    // Convert the incoming acls that we know about to qualified forms, and reject the document
if
+    // we don't know how to deal with its acls
+    Map<String,String[]> aclsMap = new HashMap<String,String[]>();
+    Map<String,String[]> denyAclsMap = new HashMap<String,String[]>();
+
+    Iterator<String> aclTypes = document.securityTypesIterator();
+    while (aclTypes.hasNext())
+    {
+      String aclType = aclTypes.next();
+      aclsMap.put(aclType,convertACL(document.getSecurityACL(aclType),authorityNameString,activities));
+      denyAclsMap.put(aclType,convertACL(document.getSecurityDenyACL(aclType),authorityNameString,activities));
+      
+      // Reject documents that have security we don't know how to deal with in the Solr plugin!!
 Only safe thing to do.
+      if (!aclType.equals(RepositoryDocument.SECURITY_TYPE_DOCUMENT) &&
+        !aclType.equals(RepositoryDocument.SECURITY_TYPE_SHARE) &&
+        !aclType.startsWith(RepositoryDocument.SECURITY_TYPE_DIRECTORY_LEVEL))
+        return false;
     }
 
     try
     {
       IngestThread t = new IngestThread(documentURI,document,arguments,keepAllMetadata,sourceTargets,
-                                        shareAcls,shareDenyAcls,directoryAclsMap,directoryDenyAclsMap,acls,denyAcls,commitWithin);
+                                        aclsMap,denyAclsMap,commitWithin);
       try
       {
         t.start();
@@ -818,12 +818,8 @@ public class HttpPoster
     protected final RepositoryDocument document;
     protected final Map<String,List<String>> arguments;
     protected final Map<String,List<String>> sourceTargets;
-    protected final String[] shareAcls;
-    protected final String[] shareDenyAcls;
-    protected final Map<Integer,String[]> directoryAclsMap;
-    protected final Map<Integer,String[]> directoryDenyAclsMap;
-    protected final String[] acls;
-    protected final String[] denyAcls;
+    protected final Map<String,String[]> aclsMap;
+    protected final Map<String,String[]> denyAclsMap;
     protected final String commitWithin;
     protected final boolean keepAllMetadata;
     
@@ -837,19 +833,16 @@ public class HttpPoster
 
     public IngestThread(String documentURI, RepositoryDocument document,
       Map<String, List<String>> arguments, boolean keepAllMetadata, Map<String,
List<String>> sourceTargets,
-      String[] shareAcls, String[] shareDenyAcls, Map<Integer,String[]> directoryAclsMap,
Map<Integer,String[]> directoryDenyAclsMap, String[] acls, String[] denyAcls, String
commitWithin)
+      Map<String,String[]> aclsMap, Map<String,String[]> denyAclsMap,
+      String commitWithin)
     {
       super();
       setDaemon(true);
       this.documentURI = documentURI;
       this.document = document;
       this.arguments = arguments;
-      this.shareAcls = shareAcls;
-      this.shareDenyAcls = shareDenyAcls;
-      this.directoryAclsMap = directoryAclsMap;
-      this.directoryDenyAclsMap = directoryDenyAclsMap;
-      this.acls = acls;
-      this.denyAcls = denyAcls;
+      this.aclsMap = aclsMap;
+      this.denyAclsMap = denyAclsMap;
       this.sourceTargets = sourceTargets;
       this.commitWithin = commitWithin;
       this.keepAllMetadata=keepAllMetadata;
@@ -912,18 +905,12 @@ public class HttpPoster
           }
           
           // Write the access token information
-          writeACLs(out,"share",shareAcls,shareDenyAcls);
-          writeACLs(out,"document",acls,denyAcls);
-
-          int directoryCount = directoryAclsMap.size();
-          int q = 0;
-          while (q < directoryCount)
+          // Both maps have the same keys.
+          Iterator<String> typeIterator = aclsMap.keySet().iterator();
+          while (typeIterator.hasNext())
           {
-            Integer index = Integer.valueOf(q);
-            String[] directoryAcls = directoryAclsMap.get(index);
-            String[] directoryDenyAcls = directoryDenyAclsMap.get(index);
-            writeACLs(out,"directory_"+index.toString(),directoryAcls,directoryDenyAcls);
-            q++;
+            String aclType = typeIterator.next();
+            writeACLs(out,aclType,aclsMap.get(aclType),denyAclsMap.get(aclType));
           }
 
           // Write the arguments

Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java?rev=1570837&r1=1570836&r2=1570837&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
(original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
Sat Feb 22 13:22:33 2014
@@ -36,16 +36,21 @@ public class RepositoryDocument
 {
   public static final String _rcsid = "@(#)$Id: RepositoryDocument.java 988245 2010-08-23
18:39:35Z kwright $";
 
+  // Security types
+  public final static String SECURITY_TYPE_DOCUMENT = "document";
+  public final static String SECURITY_TYPE_SHARE = "share";
+  public final static String SECURITY_TYPE_PARENT = "parent";
+  // Enumerated security type; add an integer to the end (deprecated)
+  public final static String SECURITY_TYPE_DIRECTORY_LEVEL = "directory_";
+  
   // Member variables.
   protected InputStream binaryFieldData = null;
   protected long binaryLength = 0;
-  protected Map<String,Object> fields = new HashMap<String,Object>();
-  protected Map<String,String[]> stringFields = new HashMap<String,String[]>();
-  protected Map<String,Reader[]> readerFields = new HashMap<String,Reader[]>();
-  protected Map<String,Date[]> dateFields = new HashMap<String,Date[]>();
-  protected Security fileSecurity = new Security();
-  protected Security shareSecurity = new Security();
-  protected List<Security> directorySecurity = new ArrayList<Security>();
+  protected final Map<String,Object> fields = new HashMap<String,Object>();
+  protected final Map<String,String[]> stringFields = new HashMap<String,String[]>();
+  protected final Map<String,Reader[]> readerFields = new HashMap<String,Reader[]>();
+  protected final Map<String,Date[]> dateFields = new HashMap<String,Date[]>();
+  protected final Map<String,Security> securityLevels = new HashMap<String,Security>();
   protected String fileName = "docname";
   protected String contentMimeType = "application/octet-stream";
   protected Date createdDate = null;
@@ -122,95 +127,202 @@ public class RepositoryDocument
     return contentMimeType;
   }
   
+  /** Locate or create a specified security level.
+  *@param securityType is the security type.
+  */
+  protected Security getSecurityLevel(String securityType)
+  {
+    Security s = securityLevels.get(securityType);
+    if (s == null)
+    {
+      s = new Security();
+      securityLevels.put(securityType, s);
+    }
+    return s;
+  }
+  
+  /** Enumerate the active security types for this document.
+  *@return an iterator over the security types.
+  */
+  public Iterator<String> securityTypesIterator()
+  {
+    return securityLevels.keySet().iterator();
+  }
+  
+  /** Set security values for a given security type.
+  *@param securityType is the security type.
+  *@param acl is the acl.
+  *@param denyAcl is the deny acl.
+  */
+  public void setSecurity(String securityType, String[] acl, String[] denyAcl)
+  {
+    if (acl != null && denyAcl != null)
+    {
+      Security s = getSecurityLevel(securityType);
+      s.setACL(acl);
+      s.setDenyACL(denyAcl);
+    }
+  }
+  
+  /** Set security acl for a given security type.
+  *@param securityType is the security type.
+  *@param acl is the acl;
+  */
+  public void setSecurityACL(String securityType, String[] acl)
+  {
+    if (acl != null)
+    {
+      Security s = getSecurityLevel(securityType);
+      s.setACL(acl);
+    }
+  }
+  
+  /** Set security deny acl for a given security type.
+  *@param securityType is the security type.
+  *@param denyAcl is the deny acl.
+  */
+  public void setSecurityDenyACL(String securityType, String[] denyAcl)
+  {
+    if (denyAcl != null)
+    {
+      Security s = getSecurityLevel(securityType);
+      s.setDenyACL(denyAcl);
+    }
+  }
+
+  /** Get security acl for a given security type.
+  *@param securityType is the security type.
+  *@return the acl, which may be null.
+  */
+  public String[] getSecurityACL(String securityType)
+  {
+    Security s = securityLevels.get(securityType);
+    if (s != null)
+      return s.getACL();
+    return null;
+  }
+
+  /** Get security deny acl for a given security type.
+  *@param securityType is the security type.
+  *@return the acl, which may be null.
+  */
+  public String[] getSecurityDenyACL(String securityType)
+  {
+    Security s = securityLevels.get(securityType);
+    if (s != null)
+      return s.getDenyACL();
+    return null;
+  }
+  
   /** Set the document's "file" allow acls.
   *@param acl is the allowed "file" access control token list for the document.
   */
+  @Deprecated
   public void setACL(String[] acl)
   {
-    fileSecurity.setACL(acl);
+    setSecurityACL(SECURITY_TYPE_DOCUMENT,acl);
   }
 
   /** Get the document's "file" allow acl, if any.
   *@return the allow access control token list for the document.
   */
+  @Deprecated
   public String[] getACL()
   {
-    return fileSecurity.getACL();
+    return getSecurityACL(SECURITY_TYPE_DOCUMENT);
   }
 
   /** Set the document's "file" deny acl.
   *@param acl is the "file" denied access control token list for the document.
   */
+  @Deprecated
   public void setDenyACL(String[] acl)
   {
-    fileSecurity.setDenyACL(acl);
+    setSecurityDenyACL(SECURITY_TYPE_DOCUMENT,acl);
   }
 
   /** Get the document's deny acl, if any.
   *@return the deny access control token list for the document.
   */
+  @Deprecated
   public String[] getDenyACL()
   {
-    return fileSecurity.getDenyACL();
+    return getSecurityDenyACL(SECURITY_TYPE_DOCUMENT);
   }
 
   /** Set document's "share" acl. */
+  @Deprecated
   public void setShareACL(String[] acl)
   {
-    shareSecurity.setACL(acl);
+    setSecurityACL(SECURITY_TYPE_SHARE,acl);
   }
 
   /** Get document's "share" acl. */
+  @Deprecated
   public String[] getShareACL()
   {
-    return shareSecurity.getACL();
+    return getSecurityACL(SECURITY_TYPE_SHARE);
   }
 
   /** Set document's "share" deny acl. */
+  @Deprecated
   public void setShareDenyACL(String[] acl)
   {
-    shareSecurity.setDenyACL(acl);
+    setSecurityDenyACL(SECURITY_TYPE_SHARE,acl);
   }
 
   /** Get document's "share" deny acl. */
+  @Deprecated
   public String[] getShareDenyACL()
   {
-    return shareSecurity.getDenyACL();
+    return getSecurityDenyACL(SECURITY_TYPE_SHARE);
   }
 
+  int directoryLevel = 0;
+
   /** Clear all directory acls. */
+  @Deprecated
   public void clearDirectoryACLs()
   {
-    directorySecurity.clear();
+    Iterator<String> securityIterator = securityLevels.keySet().iterator();
+    while (securityIterator.hasNext())
+    {
+      if (securityIterator.next().startsWith(SECURITY_TYPE_DIRECTORY_LEVEL))
+        securityIterator.remove();
+    }
+    directoryLevel = 0;
   }
 
   /** Get a count of directory security entries. */
+  @Deprecated
   public int countDirectoryACLs()
   {
-    return directorySecurity.size();
+    return directoryLevel;
   }
 
   /** Add directory security entry */
+  @Deprecated
   public void addDirectoryACLs(String[] allowACL, String[] denyACL)
   {
-    Security s = new Security();
-    s.setACL(allowACL);
-    s.setDenyACL(denyACL);
-    directorySecurity.add(s);
+    if (allowACL != null && denyACL != null)
+    {
+      setSecurity(SECURITY_TYPE_DIRECTORY_LEVEL + directoryLevel,allowACL,denyACL);
+    }
+    directoryLevel++;
   }
 
   /** Get directory security access acl */
+  @Deprecated
   public String[] getDirectoryACL(int index)
   {
-    Security s = directorySecurity.get(index);
-    return s.getACL();
+    return getSecurityACL(SECURITY_TYPE_DIRECTORY_LEVEL + index);
   }
 
   /** Get directory security deny acl */
+  @Deprecated
   public String[] getDirectoryDenyACL(int index)
   {
-    Security s = directorySecurity.get(index);
-    return s.getDenyACL();
+    return getSecurityDenyACL(SECURITY_TYPE_DIRECTORY_LEVEL + index);
   }
 
   /** Set the binary field.



Mime
View raw message