incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r930838 - in /incubator/lcf/trunk/modules/connectors/solr: connector/org/apache/lcf/agents/output/solr/ crawler-ui/output/solr/
Date Mon, 05 Apr 2010 11:00:31 GMT
Author: kwright
Date: Mon Apr  5 11:00:31 2010
New Revision: 930838

URL: http://svn.apache.org/viewvc?rev=930838&view=rev
Log:
Add the ability to configure arbitrary arguments for solr output connector.  This is apparently
how you configure pipelines in solr.

Modified:
    incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/HttpPoster.java
    incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/SolrConfig.java
    incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/SolrConnector.java
    incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/editconfig.jsp
    incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/headerconfig.jsp
    incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/postconfig.jsp
    incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/viewconfig.jsp

Modified: incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/HttpPoster.java?rev=930838&r1=930837&r2=930838&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/HttpPoster.java
(original)
+++ incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/HttpPoster.java
Mon Apr  5 11:00:31 2010
@@ -156,11 +156,14 @@ public class HttpPoster
   * Post the input stream to ingest
   * @param documentURI is the document's uri.
   * @param document is the document structure to ingest.
+  * @param arguments are the configuration arguments to pass in the post.
+  * @param activities is the activities object, so we can report what's happening.
   * @return true if the ingestion was successful, or false if the ingestion is illegal.
   * @throws LCFException, ServiceInterruption
   */
   public boolean indexPost(String documentURI,
-    RepositoryDocument document, IOutputAddActivity activities)
+    RepositoryDocument document, Map arguments,
+    IOutputAddActivity activities)
     throws LCFException, ServiceInterruption
   {
     if (Logging.ingest.isDebugEnabled())
@@ -175,7 +178,7 @@ public class HttpPoster
     {
       try
       {
-        IngestThread t = new IngestThread(documentURI,document);
+        IngestThread t = new IngestThread(documentURI,document,arguments);
         try
         {
           t.start();
@@ -770,6 +773,7 @@ public class HttpPoster
   {
     protected String documentURI;
     protected RepositoryDocument document;
+    protected Map arguments;
 
     protected Long activityStart = null;
     protected Long activityBytes = null;
@@ -779,12 +783,13 @@ public class HttpPoster
     protected boolean readFromDocumentStreamYet = false;
     protected boolean rval = false;
 
-    public IngestThread(String documentURI, RepositoryDocument document)
+    public IngestThread(String documentURI, RepositoryDocument document, Map arguments)
     {
       super();
       setDaemon(true);
       this.documentURI = documentURI;
       this.document = document;
+      this.arguments = arguments;
     }
 
     public void run()
@@ -837,8 +842,16 @@ public class HttpPoster
                 int totalLength = 0;
                 // Count the id.
                 totalLength += lengthField("literal.id",documentURI);
+                // Count the arguments
+                Iterator iter = arguments.keySet().iterator();
+                while (iter.hasNext())
+                {
+                  String name = (String)iter.next();
+                  String value = (String)arguments.get(name);
+                  totalLength += lengthField(name,value);
+                }
                 // Count the metadata.
-                Iterator iter = document.getFields();
+                iter = document.getFields();
                 while (iter.hasNext())
                 {
                   String fieldName = (String)iter.next();
@@ -875,6 +888,15 @@ public class HttpPoster
                 // Write the id field
                 writeField(out,"literal.id",documentURI);
 
+                // Write the arguments
+                iter = arguments.keySet().iterator();
+                while (iter.hasNext())
+                {
+                  String name = (String)iter.next();
+                  String value = (String)arguments.get(name);
+                  writeField(out,name,value);
+                }
+
                 // Write the metadata, each in a field by itself
                 iter = document.getFields();
                 while (iter.hasNext())

Modified: incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/SolrConfig.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/SolrConfig.java?rev=930838&r1=930837&r2=930838&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/SolrConfig.java
(original)
+++ incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/SolrConfig.java
Mon Apr  5 11:00:31 2010
@@ -47,7 +47,13 @@ public class SolrConfig
   public static final String PARAM_USERID = "User ID";
   /** Optional user password */
   public static final String PARAM_PASSWORD = "Password";
-
+  /** Node describing an argument */
+  public static final String NODE_ARGUMENT = "argument";
+  /** Attribute with the argument name */
+  public static final String ATTRIBUTE_NAME = "name";
+  /** Attribute with the argument value */
+  public static final String ATTRIBUTE_VALUE = "value";
+  
   // Output specification
 
   // Nothing yet...

Modified: incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/SolrConnector.java?rev=930838&r1=930837&r2=930838&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/SolrConnector.java
(original)
+++ incubator/lcf/trunk/modules/connectors/solr/connector/org/apache/lcf/agents/output/solr/SolrConnector.java
Mon Apr  5 11:00:31 2010
@@ -162,8 +162,45 @@ public class SolrConnector extends org.a
   public String getOutputDescription(OutputSpecification spec)
     throws LCFException
   {
-    // No output description data at this time.
-    return "";
+    // All the arguments need to go into this string, since they affect ingestion.
+    Map args = new HashMap();
+    int i = 0;
+    while (i < params.getChildCount())
+    {
+      ConfigNode node = params.getChild(i++);
+      if (node.getType().equals(SolrConfig.NODE_ARGUMENT))
+        args.put(node.getAttributeValue(SolrConfig.ATTRIBUTE_NAME),node.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE));
+    }
+    
+    String[] sortArray = new String[args.size()];
+    Iterator iter = args.keySet().iterator();
+    i = 0;
+    while (iter.hasNext())
+    {
+      sortArray[i++] = (String)iter.next();
+    }
+    
+    // Always use sorted order, because we need this to be comparable.
+    java.util.Arrays.sort(sortArray);
+    
+    String[] fixedList = new String[2];
+    ArrayList nameValues = new ArrayList();
+    i = 0;
+    while (i < sortArray.length)
+    {
+      String name = sortArray[i++];
+      String value = (String)args.get(name);
+      fixedList[0] = name;
+      fixedList[1] = value;
+      StringBuffer pairBuffer = new StringBuffer();
+      packFixedList(pairBuffer,fixedList,'=');
+      nameValues.add(pairBuffer.toString());
+    }
+    
+    StringBuffer sb = new StringBuffer();
+    packList(sb,nameValues,'+');
+    
+    return sb.toString();
   }
 
   /** Add (or replace) a document in the output data store using the connector.
@@ -183,11 +220,22 @@ public class SolrConnector extends org.a
   public int addOrReplaceDocument(String documentURI, String outputDescription, RepositoryDocument
document, String authorityNameString, IOutputAddActivity activities)
     throws LCFException, ServiceInterruption
   {
+    // Build the argument map we'll send.
+    // There's no point doing it from packed values; this is configuration based, which cannot
change or we'd get a new connection instance.
+    Map args = new HashMap();
+    int i = 0;
+    while (i < params.getChildCount())
+    {
+      ConfigNode node = params.getChild(i++);
+      if (node.getType().equals(SolrConfig.NODE_ARGUMENT))
+        args.put(node.getAttributeValue(SolrConfig.ATTRIBUTE_NAME),node.getAttributeValue(SolrConfig.ATTRIBUTE_VALUE));
+    }
+
     // Establish a session
     getSession();
 
     // Now, go off and call the ingest API.
-    if (poster.indexPost(documentURI,document,activities))
+    if (poster.indexPost(documentURI,document,args,activities))
       return DOCUMENTSTATUS_ACCEPTED;
     return DOCUMENTSTATUS_REJECTED;
   }

Modified: incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/editconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/editconfig.jsp?rev=930838&r1=930837&r2=930838&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/editconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/editconfig.jsp Mon
Apr  5 11:00:31 2010
@@ -79,7 +79,7 @@
 		password = "";
 		
 	// "Appliance" tab
-	if (tabName.equals("SOLR"))
+	if (tabName.equals("Server"))
 	{
 %>
 <table class="displaytable">
@@ -153,7 +153,7 @@
 	}
 	else
 	{
-		// SOLR tab hiddens
+		// Server tab hiddens
 %>
 <input type="hidden" name="serverprotocol" value='<%=org.apache.lcf.ui.util.Encoder.attributeEscape(protocol)%>'/>
 <input type="hidden" name="servername" value='<%=org.apache.lcf.ui.util.Encoder.attributeEscape(server)%>'/>
@@ -167,4 +167,115 @@
 <input type="hidden" name="password" value='<%=org.apache.lcf.ui.util.Encoder.attributeEscape(password)%>'/>
 <%
 	}
+
+	// Prepare for the argument tab
+	Map argumentMap = new HashMap();
+	int i = 0;
+	while (i < parameters.getChildCount())
+	{
+		ConfigNode sn = parameters.getChild(i++);
+		if (sn.getType().equals(org.apache.lcf.agents.output.solr.SolrConfig.NODE_ARGUMENT))
+		{
+			String name = sn.getAttributeValue(org.apache.lcf.agents.output.solr.SolrConfig.ATTRIBUTE_NAME);
+			String value = sn.getAttributeValue(org.apache.lcf.agents.output.solr.SolrConfig.ATTRIBUTE_VALUE);
+			argumentMap.put(name,value);
+		}
+	}
+	// "Arguments" tab
+	if (tabName.equals("Arguments"))
+	{
+		// For the display, sort the arguments into alphabetic order
+		String[] sortArray = new String[argumentMap.size()];
+		i = 0;
+		Iterator iter = argumentMap.keySet().iterator();
+		while (iter.hasNext())
+		{
+			sortArray[i++] = (String)iter.next();
+		}
+		java.util.Arrays.sort(sortArray);
+%>
+<table class="displaytable">
+	<tr><td class="separator" colspan="2"><hr/></td></tr>
+	<tr>
+		<td class="description"><nobr>Arguments:</nobr></td>
+		<td class="boxcell">
+			<table class="formtable">
+				<tr class="formheaderrow">
+					<td class="formcolumnheader"></td>
+					<td class="formcolumnheader"><nobr>Name</nobr></td>
+					<td class="formcolumnheader"><nobr>Value</nobr></td>
+				</tr>
+<%
+		i = 0;
+		while (i < sortArray.length)
+		{
+			String name = sortArray[i];
+			String value = (String)argumentMap.get(name);
+			// It's prefix will be...
+			String prefix = "argument_" + Integer.toString(i);
+%>
+				<tr class='<%=((i % 2)==0)?"evenformrow":"oddformrow"%>'>
+					<td class="formcolumncell">
+						<a name='<%=prefix%>'><input type="button" value="Delete" alt='<%="Delete
argument #"+Integer.toString(i+1)%>' onclick='<%="javascript:deleteArgument("+Integer.toString(i)+");"%>'/>
+						<input type="hidden" name='<%=prefix+"_op"%>' value="Continue"/>
+						<input type="hidden" name='<%=prefix+"_name"%>' value='<%=org.apache.lcf.ui.util.Encoder.attributeEscape(name)%>'/>
+						<input type="hidden" name='<%=prefix+"_value"%>' value='<%=org.apache.lcf.ui.util.Encoder.attributeEscape(value)%>'/>
+					</td>
+					<td class="formcolumncell">
+						<nobr><%=org.apache.lcf.ui.util.Encoder.bodyEscape(name)%></nobr>
+					</td>
+					<td class="formcolumncell">
+						<nobr><%=org.apache.lcf.ui.util.Encoder.bodyEscape(value)%></nobr>
+					</td>
+				</tr>
+<%
+			i++;
+		}
+		if (i == 0)
+		{
+%>
+				<tr class="formrow"><td class="formmessage" colspan="3">No arguments specified</td></tr>
+<%
+		}
+%>
+				<tr class="formrow"><td class="formseparator" colspan="3"><hr/></td></tr>
+				<tr class="formrow">
+					<td class="formcolumncell">
+						<a name="argument"><input type="button" value="Add" alt="Add argument" onclick="javascript:addArgument();"/></a>
+						<input type="hidden" name="argument_count" value='<%=i%>'/>
+						<input type="hidden" name="argument_op" value="Continue"/>
+					</td>
+					<td class="formcolumncell">
+						<nobr><input type="text" size="30" name="argument_name" value=""/></nobr>
+					</td>
+					<td class="formcolumncell">
+						<nobr><input type="text" size="30" name="argument_value" value=""/></nobr>
+					</td>
+				</tr>
+			</table>
+		</td>
+	</tr>
+</table>
+<%
+	}
+	else
+	{
+		// Emit hiddens for argument tab
+		i = 0;
+		Iterator iter = argumentMap.keySet().iterator();
+		while (iter.hasNext())
+		{
+			String name = (String)iter.next();
+			String value = (String)argumentMap.get(name);
+			// It's prefix will be...
+			String prefix = "argument_" + Integer.toString(i++);
+%>
+<input type="hidden" name='<%=prefix+"_name"%>' value='<%=org.apache.lcf.ui.util.Encoder.attributeEscape(name)%>'/>
+<input type="hidden" name='<%=prefix+"_value"%>' value='<%=org.apache.lcf.ui.util.Encoder.attributeEscape(value)%>'/>
+<%
+		}
+%>
+<input type="hidden" name="argument_count" value='<%=i%>'/>
+<%
+	}
 %>

Modified: incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/headerconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/headerconfig.jsp?rev=930838&r1=930837&r2=930838&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/headerconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/headerconfig.jsp Mon
Apr  5 11:00:31 2010
@@ -42,7 +42,8 @@
 	if (tabsArray == null)
 		out.println("No tabs array!");
 
-	tabsArray.add("SOLR");
+	tabsArray.add("Server");
+        tabsArray.add("Arguments");
 
 
 %>
@@ -53,13 +54,13 @@
 	{
 		if (editconnection.servername.value == "")
 		{
-			alert("Please supply a valid SOLR server name");
+			alert("Please supply a valid Solr server name");
 			editconnection.servername.focus();
 			return false;
 		}
 		if (editconnection.serverport.value != "" && !isInteger(editconnection.serverport.value))
 		{
-			alert("SOLR server port must be a valid integer");
+			alert("Solr server port must be a valid integer");
 			editconnection.serverport.focus();
 			return false;
 		}
@@ -94,49 +95,74 @@
 	{
 		if (editconnection.servername.value == "")
 		{
-			alert("Please supply a valid SOLR server name");
-			SelectTab("SOLR");
+			alert("Please supply a valid Solr server name");
+			SelectTab("Server");
 			editconnection.servername.focus();
 			return false;
 		}
 		if (editconnection.serverport.value != "" && !isInteger(editconnection.serverport.value))
 		{
-			alert("SOLR server port must be a valid integer");
-			SelectTab("SOLR");
+			alert("Solr server port must be a valid integer");
+			SelectTab("Server");
 			editconnection.serverport.focus();
 			return false;
 		}
 		if (editconnection.webappname.value != "" && editconnection.webappname.value.indexOf("/")
!= -1)
 		{
 			alert("Web application name cannot have '/' characters");
-			SelectTab("SOLR");
+			SelectTab("Server");
 			editconnection.webappname.focus();
 			return false;
 		}
 		if (editconnection.updatepath.value != "" && editconnection.updatepath.value.substring(0,1)
!= "/")
 		{
 			alert("Update path must start with a  '/' character");
-			SelectTab("SOLR");
+			SelectTab("Server");
 			editconnection.updatepath.focus();
 			return false;
 		}
 		if (editconnection.removepath.value != "" && editconnection.removepath.value.substring(0,1)
!= "/")
 		{
 			alert("Remove path must start with a  '/' character");
-			SelectTab("SOLR");
+			SelectTab("Server");
 			editconnection.removepath.focus();
 			return false;
 		}
 		if (editconnection.statuspath.value != "" && editconnection.statuspath.value.substring(0,1)
!= "/")
 		{
 			alert("Status path must start with a  '/' character");
-			SelectTab("SOLR");
+			SelectTab("Server");
 			editconnection.statuspath.focus();
 			return false;
 		}
 		return true;
 	}
 
+	function deleteArgument(i)
+	{
+		// Set the operation
+		eval("editconnection.argument_"+i+"_op.value=\"Delete\"");
+		// Submit
+		if (editconnection.argument_count.value==i)
+			postFormSetAnchor("argument");
+		else
+			postFormSetAnchor("argument_"+i)
+		// Undo, so we won't get two deletes next time
+		eval("editconnection.argument_"+i+"_op.value=\"Continue\"");
+	}
+
+	function addArgument()
+	{
+		if (editconnection.argument_name.value == "")
+		{
+			alert("Argument name cannot be an empty string");
+			editconnection.argument_name.focus();
+			return;
+		}
+		editconnection.argument_op.value="Add";
+		postFormSetAnchor("argument");
+	}
+
 //-->
 </script>
 

Modified: incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/postconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/postconfig.jsp?rev=930838&r1=930837&r2=930838&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/postconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/postconfig.jsp Mon
Apr  5 11:00:31 2010
@@ -76,5 +76,48 @@
 	String password = variableContext.getParameter("password");
 	if (password != null)
 		parameters.setObfuscatedParameter(org.apache.lcf.agents.output.solr.SolrConfig.PARAM_PASSWORD,password);
-	
+    
+	String x = variableContext.getParameter("argument_count");
+	if (x != null && x.length() > 0)
+	{
+		// About to gather the argument nodes, so get rid of the old ones.
+		int i = 0;
+		while (i < parameters.getChildCount())
+		{
+			ConfigNode node = parameters.getChild(i);
+			if (node.getType().equals(org.apache.lcf.agents.output.solr.SolrConfig.NODE_ARGUMENT))
+				parameters.removeChild(i);
+			else
+				i++;
+		}
+		int count = Integer.parseInt(x);
+		i = 0;
+		while (i < count)
+		{
+			String prefix = "argument_"+Integer.toString(i);
+			String op = variableContext.getParameter(prefix+"_op");
+			if (op == null || !op.equals("Delete"))
+			{
+				// Gather the name and value.
+				String name = variableContext.getParameter(prefix+"_name");
+				String value = variableContext.getParameter(prefix+"_value");
+				ConfigNode node = new ConfigNode(org.apache.lcf.agents.output.solr.SolrConfig.NODE_ARGUMENT);
+				node.setAttribute(org.apache.lcf.agents.output.solr.SolrConfig.ATTRIBUTE_NAME,name);
+				node.setAttribute(org.apache.lcf.agents.output.solr.SolrConfig.ATTRIBUTE_VALUE,value);
+				parameters.addChild(parameters.getChildCount(),node);
+			}
+			i++;
+		}
+		String addop = variableContext.getParameter("argument_op");
+		if (addop != null && addop.equals("Add"))
+		{
+			String name = variableContext.getParameter("argument_name");
+			String value = variableContext.getParameter("argument_value");
+			ConfigNode node = new ConfigNode(org.apache.lcf.agents.output.solr.SolrConfig.NODE_ARGUMENT);
+			node.setAttribute(org.apache.lcf.agents.output.solr.SolrConfig.ATTRIBUTE_NAME,name);
+			node.setAttribute(org.apache.lcf.agents.output.solr.SolrConfig.ATTRIBUTE_VALUE,value);
+			parameters.addChild(parameters.getChildCount(),node);
+		}
+	}
+
 %>

Modified: incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/viewconfig.jsp
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/viewconfig.jsp?rev=930838&r1=930837&r2=930838&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/viewconfig.jsp (original)
+++ incubator/lcf/trunk/modules/connectors/solr/crawler-ui/output/solr/viewconfig.jsp Mon
Apr  5 11:00:31 2010
@@ -70,4 +70,44 @@
 
 		</td>
 	</tr>
+	
+	<tr>
+		<td class="description" colspan="1"><nobr>Arguments:</nobr></td>
+		<td class="boxcell" colspan="3">
+			<table class="formtable">
+				<tr class="formheaderrow">
+					<td class="formcolumnheader"><nobr>Name</nobr></td>
+					<td class="formcolumnheader"><nobr>Value</nobr></td>
+				</tr>
+<%
+	int i = 0;
+	int instanceNumber = 0;
+	while (i < parameters.getChildCount())
+	{
+		ConfigNode cn = parameters.getChild(i++);
+		if (cn.getType().equals(org.apache.lcf.agents.output.solr.SolrConfig.NODE_ARGUMENT))
+		{
+			// An argument node!  Look for all its parameters.
+			String name = cn.getAttributeValue(org.apache.lcf.agents.output.solr.SolrConfig.ATTRIBUTE_NAME);
+			String value = cn.getAttributeValue(org.apache.lcf.agents.output.solr.SolrConfig.ATTRIBUTE_VALUE);
+%>
+				<tr class='<%=((instanceNumber % 2)==0)?"evenformrow":"oddformrow"%>'>
+					<td class="formcolumncell"><nobr><%=org.apache.lcf.ui.util.Encoder.bodyEscape(name)%></nobr></td>
+					<td class="formcolumncell"><nobr><%=org.apache.lcf.ui.util.Encoder.bodyEscape(value)%></nobr></td>
+				</tr>
+<%
+			instanceNumber++;
+		}
+	}
+	if (instanceNumber == 0)
+	{
+%>
+				<tr class="formrow"><td class="formmessage" colspan="5">No arguments</td></tr>
+<%
+	}
+%>
+			</table>
+		</td>
+	</tr>
+
 </table>



Mime
View raw message