manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1452563 - in /manifoldcf/trunk/connectors/solr/connector/src/main: java/org/apache/manifoldcf/agents/output/solr/ native2ascii/org/apache/manifoldcf/agents/output/solr/
Date Mon, 04 Mar 2013 22:16:09 GMT
Author: kwright
Date: Mon Mar  4 22:16:08 2013
New Revision: 1452563

URL: http://svn.apache.org/r1452563
Log:
Add common field support to Solr connector.  Part of CONNECTORS-657.

Modified:
    manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
    manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
    manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
    manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
    manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1452563&r1=1452562&r2=1452563&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Mon Mar  4 22:16:08 2013
@@ -103,6 +103,10 @@ public class HttpPoster
   private String allowAttributeName;
   private String denyAttributeName;
   private String idAttributeName;
+  private String modifiedDateAttributeName;
+  private String createdDateAttributeName;
+  private String fileNameAttributeName;
+  private String mimeTypeAttributeName;
   
   // Document max length
   private Long maxDocumentLength;
@@ -125,6 +129,8 @@ public class HttpPoster
     int zkClientTimeout, int zkConnectTimeout,
     String updatePath, String removePath, String statusPath,
     String allowAttributeName, String denyAttributeName, String idAttributeName,
+    String modifiedDateAttributeName, String createdDateAttributeName,
+    String fileNameAttributeName, String mimeTypeAttributeName,
     Long maxDocumentLength,
     String commitWithin)
     throws ManifoldCFException
@@ -139,6 +145,10 @@ public class HttpPoster
     this.allowAttributeName = allowAttributeName;
     this.denyAttributeName = denyAttributeName;
     this.idAttributeName = idAttributeName;
+    this.modifiedDateAttributeName = modifiedDateAttributeName;
+    this.createdDateAttributeName = createdDateAttributeName;
+    this.fileNameAttributeName = fileNameAttributeName;
+    this.mimeTypeAttributeName = mimeTypeAttributeName;
     
     this.maxDocumentLength = maxDocumentLength;
     
@@ -164,6 +174,8 @@ public class HttpPoster
     String updatePath, String removePath, String statusPath,
     String realm, String userID, String password,
     String allowAttributeName, String denyAttributeName, String idAttributeName,
+    String modifiedDateAttributeName, String createdDateAttributeName,
+    String fileNameAttributeName, String mimeTypeAttributeName,
     IKeystoreManager keystoreManager, Long maxDocumentLength,
     String commitWithin)
     throws ManifoldCFException
@@ -178,6 +190,10 @@ public class HttpPoster
     this.allowAttributeName = allowAttributeName;
     this.denyAttributeName = denyAttributeName;
     this.idAttributeName = idAttributeName;
+    this.modifiedDateAttributeName = modifiedDateAttributeName;
+    this.createdDateAttributeName = createdDateAttributeName;
+    this.fileNameAttributeName = fileNameAttributeName;
+    this.mimeTypeAttributeName = mimeTypeAttributeName;
     
     this.maxDocumentLength = maxDocumentLength;
 
@@ -782,7 +798,34 @@ public class HttpPoster
           
           // Write the id field
           writeField(out,LITERAL+idAttributeName,documentURI);
-
+          // Write the rest of the attributes
+          if (modifiedDateAttributeName != null)
+          {
+            Date date = document.getModifiedDate();
+            if (date != null)
+              // Write value
+              writeField(out,LITERAL+modifiedDateAttributeName,convertToISO(date));
+          }
+          if (createdDateAttributeName != null)
+          {
+            Date date = document.getCreatedDate();
+            if (date != null)
+              // Write value
+              writeField(out,LITERAL+createdDateAttributeName,convertToISO(date));
+          }
+          if (fileNameAttributeName != null)
+          {
+            String fileName = document.getFileName();
+            if (fileName != null)
+              writeField(out,LITERAL+fileNameAttributeName,fileName);
+          }
+          if (mimeTypeAttributeName != null)
+          {
+            String mimeType = document.getMimeType();
+            if (mimeType != null)
+              writeField(out,LITERAL+mimeTypeAttributeName,mimeType);
+          }
+          
           // Write the access token information
           writeACLs(out,"share",shareAcls,shareDenyAcls);
           writeACLs(out,"document",acls,denyAcls);
@@ -935,6 +978,12 @@ public class HttpPoster
     }
   }
 
+  protected static String convertToISO(Date date)
+  {
+    java.text.DateFormat df = new java.text.SimpleDateFormat("yyyy-MM-dd'T'HH:mmZ");
+    return df.format(date);
+  }
+  
   /** Killable thread that does deletions.
   * Java 1.5 stopped permitting thread interruptions to abort socket waits.  As a result,
it is impossible to get threads to shutdown cleanly that are doing
   * such waits.  So, the places where this happens are segregated in their own threads so
that they can be just abandoned.

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java?rev=1452563&r1=1452562&r2=1452563&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
(original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
Mon Mar  4 22:16:08 2013
@@ -83,6 +83,14 @@ public class SolrConfig
   public static final String PARAM_STATUSPATH = "Server status handler";
   /** Id field */
   public static final String PARAM_IDFIELD = "Solr id field name";
+  /** Optional modified date field */
+  public static final String PARAM_MODIFIEDDATEFIELD = "Solr modified date field name";
+  /** Optional created date field */
+  public static final String PARAM_CREATEDDATEFIELD = "Solr created date field name";
+  /** Optional file name field */
+  public static final String PARAM_FILENAMEFIELD = "Solr filename field name";
+  /** Optional mime type field */
+  public static final String PARAM_MIMETYPEFIELD = "Solr mime type field name";
   /** Optional basic auth realm */
   public static final String PARAM_REALM = "Realm";
   /** Optional user ID */

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1452563&r1=1452562&r2=1452563&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
(original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Mon Mar  4 22:16:08 2013
@@ -151,7 +151,23 @@ public class SolrConnector extends org.a
       String idAttributeName = params.getParameter(SolrConfig.PARAM_IDFIELD);
       if (idAttributeName == null || idAttributeName.length() == 0)
         idAttributeName = "id";
-      
+
+      String modifiedDateAttributeName = params.getParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD);
+      if (modifiedDateAttributeName == null || modifiedDateAttributeName.length() == 0)
+        modifiedDateAttributeName = null;
+
+      String createdDateAttributeName = params.getParameter(SolrConfig.PARAM_CREATEDDATEFIELD);
+      if (createdDateAttributeName == null || createdDateAttributeName.length() == 0)
+        createdDateAttributeName = null;
+  
+      String fileNameAttributeName = params.getParameter(SolrConfig.PARAM_FILENAMEFIELD);
+      if (fileNameAttributeName == null || fileNameAttributeName.length() == 0)
+        fileNameAttributeName = null;
+
+      String mimeTypeAttributeName = params.getParameter(SolrConfig.PARAM_MIMETYPEFIELD);
+      if (mimeTypeAttributeName == null || mimeTypeAttributeName.length() == 0)
+        mimeTypeAttributeName = null;
+
       String commits = params.getParameter(SolrConfig.PARAM_COMMITS);
       if (commits == null || commits.length() == 0)
         commits = "true";
@@ -257,6 +273,8 @@ public class SolrConnector extends org.a
             connectTimeout,socketTimeout,
             updatePath,removePath,statusPath,realm,userID,password,
             allowAttributeName,denyAttributeName,idAttributeName,
+            modifiedDateAttributeName,createdDateAttributeName,
+            fileNameAttributeName,mimeTypeAttributeName,
             keystoreManager,maxDocumentLength,commitWithin);
           
         }
@@ -310,6 +328,8 @@ public class SolrConnector extends org.a
             zkClientTimeout,zkConnectTimeout,
             updatePath,removePath,statusPath,
             allowAttributeName,denyAttributeName,idAttributeName,
+            modifiedDateAttributeName,createdDateAttributeName,
+            fileNameAttributeName,mimeTypeAttributeName,
             maxDocumentLength,commitWithin);
           
         }
@@ -1032,6 +1052,22 @@ public class SolrConnector extends org.a
     if (idField == null)
       idField = "id";
     
+    String modifiedDateField = parameters.getParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD);
+    if (modifiedDateField == null)
+      modifiedDateField = "";
+    
+    String createdDateField = parameters.getParameter(SolrConfig.PARAM_CREATEDDATEFIELD);
+    if (createdDateField == null)
+      createdDateField = "";
+    
+    String fileNameField = parameters.getParameter(SolrConfig.PARAM_FILENAMEFIELD);
+    if (fileNameField == null)
+      fileNameField = "";
+    
+    String mimeTypeField = parameters.getParameter(SolrConfig.PARAM_MIMETYPEFIELD);
+    if (mimeTypeField == null)
+      mimeTypeField = "";
+    
     String realm = parameters.getParameter(SolrConfig.PARAM_REALM);
     if (realm == null)
       realm = "";
@@ -1453,13 +1489,41 @@ public class SolrConnector extends org.a
 "      <input name=\"idfield\" type=\"text\" size=\"32\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(idField)+"\"/>\n"+
 "    </td>\n"+
 "  </tr>\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"SolrConnector.ModifiedDateFieldName")
+ "</nobr></td>\n"+
+"    <td class=\"value\">\n"+
+"      <input name=\"modifieddatefield\" type=\"text\" size=\"32\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(modifiedDateField)+"\"/>\n"+
+"    </td>\n"+
+"  </tr>\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"SolrConnector.CreatedDateFieldName")
+ "</nobr></td>\n"+
+"    <td class=\"value\">\n"+
+"      <input name=\"createddatefield\" type=\"text\" size=\"32\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(createdDateField)+"\"/>\n"+
+"    </td>\n"+
+"  </tr>\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"SolrConnector.FileNameFieldName")
+ "</nobr></td>\n"+
+"    <td class=\"value\">\n"+
+"      <input name=\"filenamefield\" type=\"text\" size=\"32\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(fileNameField)+"\"/>\n"+
+"    </td>\n"+
+"  </tr>\n"+
+"  <tr>\n"+
+"    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"SolrConnector.MimeTypeFieldName")
+ "</nobr></td>\n"+
+"    <td class=\"value\">\n"+
+"      <input name=\"mimetypefield\" type=\"text\" size=\"32\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(mimeTypeField)+"\"/>\n"+
+"    </td>\n"+
+"  </tr>\n"+
 "</table>\n"
       );
     }
     else
     {
       out.print(
-"<input type=\"hidden\" name=\"idfield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(idField)+"\"/>\n"
+"<input type=\"hidden\" name=\"idfield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(idField)+"\"/>\n"+
+"<input type=\"hidden\" name=\"modifieddatefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(modifiedDateField)+"\"/>\n"+
+"<input type=\"hidden\" name=\"createddatefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(createdDateField)+"\"/>\n"+
+"<input type=\"hidden\" name=\"filenamefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(fileNameField)+"\"/>\n"+
+"<input type=\"hidden\" name=\"mimetypefield\" value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(mimeTypeField)+"\"/>\n"
       );
     }
     
@@ -1739,6 +1803,22 @@ public class SolrConnector extends org.a
     if (idField != null)
       parameters.setParameter(SolrConfig.PARAM_IDFIELD,idField);
 
+    String modifiedDateField = variableContext.getParameter("modifieddatefield");
+    if (modifiedDateField != null)
+      parameters.setParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD,modifiedDateField);
+
+    String createdDateField = variableContext.getParameter("createddatefield");
+    if (createdDateField != null)
+      parameters.setParameter(SolrConfig.PARAM_CREATEDDATEFIELD,createdDateField);
+
+    String fileNameField = variableContext.getParameter("filenamefield");
+    if (fileNameField != null)
+      parameters.setParameter(SolrConfig.PARAM_FILENAMEFIELD,fileNameField);
+
+    String mimeTypeField = variableContext.getParameter("mimetypefield");
+    if (mimeTypeField != null)
+      parameters.setParameter(SolrConfig.PARAM_MIMETYPEFIELD,mimeTypeField);
+
     String realm = variableContext.getParameter("realm");
     if (realm != null)
       parameters.setParameter(SolrConfig.PARAM_REALM,realm);

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties?rev=1452563&r1=1452562&r2=1452563&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
(original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
Mon Mar  4 22:16:08 2013
@@ -52,6 +52,10 @@ SolrConnector.Add=Add
 SolrConnector.AddZookeeperHost=Add ZooKeeper host
 SolrConnector.Certificate=Certificate:
 SolrConnector.IDFieldName=ID field name:
+SolrConnector.ModifiedDateFieldName=Modified date field name:
+SolrConnector.CreatedDateFieldName=Created date field name:
+SolrConnector.FileNameFieldName=File name field name:
+SolrConnector.MimeTypeFieldName=Mime type field name:
 SolrConnector.MaximumDocumentLength=Maximum document length:
 SolrConnector.IncludedMimeTypes=Included mime types:
 SolrConnector.ExcludedMimeTypes=Excluded mime types:

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties?rev=1452563&r1=1452562&r2=1452563&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
(original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
Mon Mar  4 22:16:08 2013
@@ -52,6 +52,10 @@ SolrConnector.Add=追加
 SolrConnector.AddZookeeperHost=ZooKeeperホストを追加
 SolrConnector.Certificate=証明証:
 SolrConnector.IDFieldName=IDフィールド名:
+SolrConnector.ModifiedDateFieldName=Modified date field name:
+SolrConnector.CreatedDateFieldName=Created date field name:
+SolrConnector.FileNameFieldName=File name field name:
+SolrConnector.MimeTypeFieldName=Mime type field name:
 SolrConnector.MaximumDocumentLength=最大コンテンツ長:
 SolrConnector.IncludedMimeTypes=含むMIMEタイプ:
 SolrConnector.ExcludedMimeTypes=除外するMIMEタイプ:



Mime
View raw message