chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r1614808 [7/7] - in /chukwa/trunk: ./ conf/ contrib/solr/ contrib/solr/logs/ contrib/solr/logs/conf/ contrib/solr/logs/conf/clustering/ contrib/solr/logs/conf/clustering/carrot2/ contrib/solr/logs/conf/lang/ contrib/solr/logs/conf/velocity/...
Date Thu, 31 Jul 2014 04:05:02 GMT
Added: chukwa/trunk/contrib/solr/logs/conf/xslt/luke.xsl
URL: http://svn.apache.org/viewvc/chukwa/trunk/contrib/solr/logs/conf/xslt/luke.xsl?rev=1614808&view=auto
==============================================================================
--- chukwa/trunk/contrib/solr/logs/conf/xslt/luke.xsl (added)
+++ chukwa/trunk/contrib/solr/logs/conf/xslt/luke.xsl Thu Jul 31 04:04:59 2014
@@ -0,0 +1,337 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+    
+    http://www.apache.org/licenses/LICENSE-2.0
+    
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+
+
+<!-- 
+  Display the luke request handler with graphs
+ -->
+<xsl:stylesheet
+    xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+    xmlns="http://www.w3.org/1999/xhtml"
+    version="1.0"
+    >
+    <xsl:output
+        method="html"
+        encoding="UTF-8"
+        media-type="text/html"
+        doctype-public="-//W3C//DTD XHTML 1.0 Strict//EN"
+        doctype-system="http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"
+    />
+
+    <xsl:variable name="title">Solr Luke Request Handler Response</xsl:variable>
+
+    <xsl:template match="/">
+        <html xmlns="http://www.w3.org/1999/xhtml">
+            <head>
+                <link rel="stylesheet" type="text/css" href="solr-admin.css"/>
+                <link rel="icon" href="favicon.ico" type="image/ico"/>
+                <link rel="shortcut icon" href="favicon.ico" type="image/ico"/>
+                <title>
+                    <xsl:value-of select="$title"/>
+                </title>
+                <xsl:call-template name="css"/>
+
+            </head>
+            <body>
+                <h1>
+                    <xsl:value-of select="$title"/>
+                </h1>
+                <div class="doc">
+                    <ul>
+                        <xsl:if test="response/lst[@name='index']">
+                            <li>
+                                <a href="#index">Index Statistics</a>
+                            </li>
+                        </xsl:if>
+                        <xsl:if test="response/lst[@name='fields']">
+                            <li>
+                                <a href="#fields">Field Statistics</a>
+                                <ul>
+                                    <xsl:for-each select="response/lst[@name='fields']/lst">
+                                        <li>
+                                            <a href="#{@name}">
+                                                <xsl:value-of select="@name"/>
+                                            </a>
+                                        </li>
+                                    </xsl:for-each>
+                                </ul>
+                            </li>
+                        </xsl:if>
+                        <xsl:if test="response/lst[@name='doc']">
+                            <li>
+                                <a href="#doc">Document statistics</a>
+                            </li>
+                        </xsl:if>
+                    </ul>
+                </div>
+                <xsl:if test="response/lst[@name='index']">
+                    <h2><a name="index"/>Index Statistics</h2>
+                    <xsl:apply-templates select="response/lst[@name='index']"/>
+                </xsl:if>
+                <xsl:if test="response/lst[@name='fields']">
+                    <h2><a name="fields"/>Field Statistics</h2>
+                    <xsl:apply-templates select="response/lst[@name='fields']"/>
+                </xsl:if>
+                <xsl:if test="response/lst[@name='doc']">
+                    <h2><a name="doc"/>Document statistics</h2>
+                    <xsl:apply-templates select="response/lst[@name='doc']"/>
+                </xsl:if>
+            </body>
+        </html>
+    </xsl:template>
+
+    <xsl:template match="lst">
+        <xsl:if test="parent::lst">
+            <tr>
+                <td colspan="2">
+                    <div class="doc">
+                        <xsl:call-template name="list"/>
+                    </div>
+                </td>
+            </tr>
+        </xsl:if>
+        <xsl:if test="not(parent::lst)">
+            <div class="doc">
+                <xsl:call-template name="list"/>
+            </div>
+        </xsl:if>
+    </xsl:template>
+
+    <xsl:template name="list">
+        <xsl:if test="count(child::*)>0">
+            <table>
+                <thead>
+                    <tr>
+                        <th colspan="2">
+                            <p>
+                                <a name="{@name}"/>
+                            </p>
+                            <xsl:value-of select="@name"/>
+                        </th>
+                    </tr>
+                </thead>
+                <tbody>
+                    <xsl:choose>
+                        <xsl:when
+                            test="@name='histogram'">
+                            <tr>
+                                <td colspan="2">
+                                    <xsl:call-template name="histogram"/>
+                                </td>
+                            </tr>
+                        </xsl:when>
+                        <xsl:otherwise>
+                            <xsl:apply-templates/>
+                        </xsl:otherwise>
+                    </xsl:choose>
+                </tbody>
+            </table>
+        </xsl:if>
+    </xsl:template>
+
+    <xsl:template name="histogram">
+        <div class="doc">
+            <xsl:call-template name="barchart">
+                <xsl:with-param name="max_bar_width">50</xsl:with-param>
+                <xsl:with-param name="iwidth">800</xsl:with-param>
+                <xsl:with-param name="iheight">160</xsl:with-param>
+                <xsl:with-param name="fill">blue</xsl:with-param>
+            </xsl:call-template>
+        </div>
+    </xsl:template>
+
+    <xsl:template name="barchart">
+        <xsl:param name="max_bar_width"/>
+        <xsl:param name="iwidth"/>
+        <xsl:param name="iheight"/>
+        <xsl:param name="fill"/>
+        <xsl:variable name="max">
+            <xsl:for-each select="int">
+                <xsl:sort data-type="number" order="descending"/>
+                <xsl:if test="position()=1">
+                    <xsl:value-of select="."/>
+                </xsl:if>
+            </xsl:for-each>
+        </xsl:variable>
+        <xsl:variable name="bars">
+           <xsl:value-of select="count(int)"/>
+        </xsl:variable>
+        <xsl:variable name="bar_width">
+           <xsl:choose>
+             <xsl:when test="$max_bar_width &lt; ($iwidth div $bars)">
+               <xsl:value-of select="$max_bar_width"/>
+             </xsl:when>
+             <xsl:otherwise>
+               <xsl:value-of select="$iwidth div $bars"/>
+             </xsl:otherwise>
+           </xsl:choose>
+        </xsl:variable>
+        <table class="histogram">
+           <tbody>
+              <tr>
+                <xsl:for-each select="int">
+                   <td>
+                 <xsl:value-of select="."/>
+                 <div class="histogram">
+                  <xsl:attribute name="style">background-color: <xsl:value-of select="$fill"/>;
width: <xsl:value-of select="$bar_width"/>px; height: <xsl:value-of select="($iheight*number(.))
div $max"/>px;</xsl:attribute>
+                 </div>
+                   </td> 
+                </xsl:for-each>
+              </tr>
+              <tr>
+                <xsl:for-each select="int">
+                   <td>
+                       <xsl:value-of select="@name"/>
+                   </td>
+                </xsl:for-each>
+              </tr>
+           </tbody>
+        </table>
+    </xsl:template>
+
+    <xsl:template name="keyvalue">
+        <xsl:choose>
+            <xsl:when test="@name">
+                <tr>
+                    <td class="name">
+                        <xsl:value-of select="@name"/>
+                    </td>
+                    <td class="value">
+                        <xsl:value-of select="."/>
+                    </td>
+                </tr>
+            </xsl:when>
+            <xsl:otherwise>
+                <xsl:value-of select="."/>
+            </xsl:otherwise>
+        </xsl:choose>
+    </xsl:template>
+
+    <xsl:template match="int|bool|long|float|double|uuid|date">
+        <xsl:call-template name="keyvalue"/>
+    </xsl:template>
+
+    <xsl:template match="arr">
+        <tr>
+            <td class="name">
+                <xsl:value-of select="@name"/>
+            </td>
+            <td class="value">
+                <ul>
+                    <xsl:for-each select="child::*">
+                        <li>
+                            <xsl:apply-templates/>
+                        </li>
+                    </xsl:for-each>
+                </ul>
+            </td>
+        </tr>
+    </xsl:template>
+
+    <xsl:template match="str">
+        <xsl:choose>
+            <xsl:when test="@name='schema' or @name='index' or @name='flags'">
+                <xsl:call-template name="schema"/>
+            </xsl:when>
+            <xsl:otherwise>
+                <xsl:call-template name="keyvalue"/>
+            </xsl:otherwise>
+        </xsl:choose>
+    </xsl:template>
+
+    <xsl:template name="schema">
+        <tr>
+            <td class="name">
+                <xsl:value-of select="@name"/>
+            </td>
+            <td class="value">
+                <xsl:if test="contains(.,'unstored')">
+                    <xsl:value-of select="."/>
+                </xsl:if>
+                <xsl:if test="not(contains(.,'unstored'))">
+                    <xsl:call-template name="infochar2string">
+                        <xsl:with-param name="charList">
+                            <xsl:value-of select="."/>
+                        </xsl:with-param>
+                    </xsl:call-template>
+                </xsl:if>
+            </td>
+        </tr>
+    </xsl:template>
+
+    <xsl:template name="infochar2string">
+        <xsl:param name="i">1</xsl:param>
+        <xsl:param name="charList"/>
+
+        <xsl:variable name="char">
+            <xsl:value-of select="substring($charList,$i,1)"/>
+        </xsl:variable>
+        <xsl:choose>
+            <xsl:when test="$char='I'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='I']"/>
- </xsl:when>
+            <xsl:when test="$char='T'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='T']"/>
- </xsl:when>
+            <xsl:when test="$char='S'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='S']"/>
- </xsl:when>
+            <xsl:when test="$char='M'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='M']"/>
- </xsl:when>
+            <xsl:when test="$char='V'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='V']"/>
- </xsl:when>
+            <xsl:when test="$char='o'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='o']"/>
- </xsl:when>
+            <xsl:when test="$char='p'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='p']"/>
- </xsl:when>
+            <xsl:when test="$char='O'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='O']"/>
- </xsl:when>
+            <xsl:when test="$char='L'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='L']"/>
- </xsl:when>
+            <xsl:when test="$char='B'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='B']"/>
- </xsl:when>
+            <xsl:when test="$char='C'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='C']"/>
- </xsl:when>
+            <xsl:when test="$char='f'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='f']"/>
- </xsl:when>
+            <xsl:when test="$char='l'">
+                <xsl:value-of select="/response/lst[@name='info']/lst/str[@name='l']"/>
-
+            </xsl:when>
+        </xsl:choose>
+
+        <xsl:if test="not($i>=string-length($charList))">
+            <xsl:call-template name="infochar2string">
+                <xsl:with-param name="i">
+                    <xsl:value-of select="$i+1"/>
+                </xsl:with-param>
+                <xsl:with-param name="charList">
+                    <xsl:value-of select="$charList"/>
+                </xsl:with-param>
+            </xsl:call-template>
+        </xsl:if>
+    </xsl:template>
+    <xsl:template name="css">
+        <style type="text/css">
+            <![CDATA[
+            td.name {font-style: italic; font-size:80%; }
+            .doc { margin: 0.5em; border: solid grey 1px; }
+            .exp { display: none; font-family: monospace; white-space: pre; }
+            div.histogram { background: none repeat scroll 0%; -moz-background-clip: -moz-initial;
-moz-background-origin: -moz-initial; -moz-background-inline-policy: -moz-initial;}
+            table.histogram { width: auto; vertical-align: bottom; }
+            table.histogram td, table.histogram th { text-align: center; vertical-align:
bottom; border-bottom: 1px solid #ff9933; width: auto; }
+            ]]>
+        </style>
+    </xsl:template>
+</xsl:stylesheet>

Added: chukwa/trunk/contrib/solr/logs/conf/xslt/updateXml.xsl
URL: http://svn.apache.org/viewvc/chukwa/trunk/contrib/solr/logs/conf/xslt/updateXml.xsl?rev=1614808&view=auto
==============================================================================
--- chukwa/trunk/contrib/solr/logs/conf/xslt/updateXml.xsl (added)
+++ chukwa/trunk/contrib/solr/logs/conf/xslt/updateXml.xsl Thu Jul 31 04:04:59 2014
@@ -0,0 +1,70 @@
+<!-- 
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+<!--
+  Simple transform of Solr query response into Solr Update XML compliant XML.
+  When used in the xslt response writer you will get UpdaateXML as output.
+  But you can also store a query response XML to disk and feed this XML to
+  the XSLTUpdateRequestHandler to index the content. Provided as example only.
+  See http://wiki.apache.org/solr/XsltUpdateRequestHandler for more info
+ -->
+<xsl:stylesheet version='1.0' xmlns:xsl='http://www.w3.org/1999/XSL/Transform'>
+  <xsl:output media-type="text/xml" method="xml" indent="yes"/>
+
+  <xsl:template match='/'>
+    <add>
+        <xsl:apply-templates select="response/result/doc"/>
+    </add>
+  </xsl:template>
+  
+  <!-- Ignore score (makes no sense to index) -->
+  <xsl:template match="doc/*[@name='score']" priority="100">
+  </xsl:template>
+
+  <xsl:template match="doc">
+    <xsl:variable name="pos" select="position()"/>
+    <doc>
+        <xsl:apply-templates>
+          <xsl:with-param name="pos"><xsl:value-of select="$pos"/></xsl:with-param>
+        </xsl:apply-templates>
+    </doc>
+  </xsl:template>
+
+  <!-- Flatten arrays to duplicate field lines -->
+  <xsl:template match="doc/arr" priority="100">
+      <xsl:variable name="fn" select="@name"/>
+      
+      <xsl:for-each select="*">
+		<xsl:element name="field">
+		    <xsl:attribute name="name"><xsl:value-of select="$fn"/></xsl:attribute>
+	        <xsl:value-of select="."/>
+		</xsl:element>
+      </xsl:for-each>
+  </xsl:template>
+
+
+  <xsl:template match="doc/*">
+      <xsl:variable name="fn" select="@name"/>
+
+	<xsl:element name="field">
+	    <xsl:attribute name="name"><xsl:value-of select="$fn"/></xsl:attribute>
+        <xsl:value-of select="."/>
+	</xsl:element>
+  </xsl:template>
+
+  <xsl:template match="*"/>
+</xsl:stylesheet>

Added: chukwa/trunk/contrib/solr/logs/core.properties
URL: http://svn.apache.org/viewvc/chukwa/trunk/contrib/solr/logs/core.properties?rev=1614808&view=auto
==============================================================================
--- chukwa/trunk/contrib/solr/logs/core.properties (added)
+++ chukwa/trunk/contrib/solr/logs/core.properties Thu Jul 31 04:04:59 2014
@@ -0,0 +1 @@
+name=logs

Modified: chukwa/trunk/pom.xml
URL: http://svn.apache.org/viewvc/chukwa/trunk/pom.xml?rev=1614808&r1=1614807&r2=1614808&view=diff
==============================================================================
--- chukwa/trunk/pom.xml (original)
+++ chukwa/trunk/pom.xml Thu Jul 31 04:04:59 2014
@@ -300,6 +300,33 @@
             <version>1.2.1</version>
             <scope>test</scope>
           </dependency>
+          <dependency>
+            <groupId>org.apache.solr</groupId>
+            <artifactId>solr-solrj</artifactId>
+            <version>4.9.0</version>
+            <type>jar</type>
+          </dependency>
+          <dependency>
+            <groupId>org.apache.solr</groupId>
+            <artifactId>solr-test-framework</artifactId>
+            <version>4.9.0</version>
+            <type>jar</type>
+            <scope>test</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-auth</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-common</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-hdfs</artifactId>
+              </exclusion>
+            </exclusions>
+          </dependency>
     </dependencies>
 
     <developers>
@@ -1239,7 +1266,7 @@
             <artifactId>maven-jxr-plugin</artifactId>
             <version>2.3</version>
         </plugin>
-<!--        <plugin>
+        <plugin>
             <artifactId>maven-pmd-plugin</artifactId>
             <version>2.6</version>
             <reportSets>
@@ -1262,7 +1289,7 @@
                 <threshold>Normal</threshold>
                 <effort>Max</effort>
             </configuration>
-        </plugin>-->
+        </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-project-info-reports-plugin</artifactId>

Added: chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/solr/SolrWriter.java
URL: http://svn.apache.org/viewvc/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/solr/SolrWriter.java?rev=1614808&view=auto
==============================================================================
--- chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/solr/SolrWriter.java
(added)
+++ chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/solr/SolrWriter.java
Thu Jul 31 04:04:59 2014
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.chukwa.datacollection.writer.solr;
+
+import java.util.List;
+
+import org.apache.hadoop.chukwa.Chunk;
+import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
+import org.apache.hadoop.chukwa.datacollection.writer.ChukwaWriter;
+import org.apache.hadoop.chukwa.datacollection.writer.PipelineableWriter;
+import org.apache.hadoop.chukwa.datacollection.writer.WriterException;
+import org.apache.hadoop.chukwa.util.ExceptionUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
+import org.apache.solr.common.SolrInputDocument;
+
+public class SolrWriter extends PipelineableWriter {
+  private static Logger log = Logger.getLogger(SolrWriter.class);
+  private static CloudSolrServer server;
+  private static String ID = "id";
+  private static String SEQ_ID = "seqId";
+  private static String DATA_TYPE = "type";
+  private static String STREAM_NAME = "stream";
+  private static String TAGS = "tags";
+  private static String SOURCE = "source";
+  private static String DATA = "data";
+
+  public SolrWriter() throws WriterException {
+    init(ChukwaAgent.getStaticConfiguration());
+  }
+  
+  @Override
+  public void init(Configuration c) throws WriterException {
+    String serverName = c.get("solr.cloud.address");
+    if (serverName == null) {
+      throw new WriterException("Solr server address is not defined.");
+    }
+    String collection = c.get("solr.collection", "logs");
+    server = new CloudSolrServer(serverName);
+    server.setDefaultCollection(collection);
+  }
+
+  @Override
+  public void close() throws WriterException {
+  }
+
+  @Override
+  public CommitStatus add(List<Chunk> chunks) throws WriterException {
+    CommitStatus rv = ChukwaWriter.COMMIT_OK;
+    for(Chunk chunk : chunks) {
+      try {
+        SolrInputDocument doc = new SolrInputDocument();
+        doc.addField(ID, chunk.getSource() + "_" + chunk.getSeqID());
+        doc.addField(TAGS, chunk.getTags());
+        doc.addField(STREAM_NAME, chunk.getStreamName());
+        doc.addField(SOURCE, chunk.getSource());
+        doc.addField(SEQ_ID, chunk.getSeqID());
+        doc.addField(DATA_TYPE, chunk.getDataType());
+        doc.addField(DATA, new String(chunk.getData()));
+        server.add(doc);
+        server.commit();
+      } catch (Exception e) {
+        log.error(ExceptionUtil.getStackTrace(e));
+        throw new WriterException("Failed to store data to Solr Cloud.");
+      }
+    }
+    if (next != null) {
+      rv = next.add(chunks); //pass data through
+    }
+    return rv;
+  }
+}

Modified: chukwa/trunk/src/packages/tarball/all.xml
URL: http://svn.apache.org/viewvc/chukwa/trunk/src/packages/tarball/all.xml?rev=1614808&r1=1614807&r2=1614808&view=diff
==============================================================================
--- chukwa/trunk/src/packages/tarball/all.xml (original)
+++ chukwa/trunk/src/packages/tarball/all.xml Thu Jul 31 04:04:59 2014
@@ -36,6 +36,10 @@
       <outputDirectory>etc/chukwa</outputDirectory>
     </fileSet>
     <fileSet>
+      <directory>contrib/solr</directory>
+      <outputDirectory>etc/solr</outputDirectory>
+    </fileSet>
+    <fileSet>
       <directory>bin</directory>
       <includes>
         <include>chukwa</include>

Added: chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/solr/TestSolrWriter.java
URL: http://svn.apache.org/viewvc/chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/solr/TestSolrWriter.java?rev=1614808&view=auto
==============================================================================
--- chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/solr/TestSolrWriter.java
(added)
+++ chukwa/trunk/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/solr/TestSolrWriter.java
Thu Jul 31 04:04:59 2014
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.chukwa.datacollection.writer.solr;
+
+import java.util.ArrayList;
+
+import org.apache.hadoop.chukwa.Chunk;
+import org.apache.hadoop.chukwa.ChunkImpl;
+import org.apache.hadoop.chukwa.util.ExceptionUtil;
+import org.apache.log4j.Logger;
+import org.apache.solr.SolrJettyTestBase;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.core.CoreContainer;
+
+import junit.framework.Assert;
+
+public class TestSolrWriter extends SolrJettyTestBase {
+  private static Logger log = Logger.getLogger(TestSolrWriter.class);
+  CoreContainer container;
+  
+  public void setUp() {
+    try {
+      String dataDir = System.getProperty("CHUKWA_DATA_DIR", "target/test/var");
+      container = new CoreContainer(dataDir);
+      container.load();
+
+      server = new EmbeddedSolrServer(container, "collection1" );
+      super.setUp();
+    } catch (Exception e) {
+      log.error(ExceptionUtil.getStackTrace(e));
+      Assert.fail(e.getMessage());
+    }    
+  }
+  
+  public void tearDown() throws Exception {
+    if (server != null) {
+      server.shutdown();
+    }
+    super.tearDown();
+  }
+  
+  /**
+   * Test adding a chunk to solr cloud, then query for the same chunk.
+   */
+  public void testCommit() {
+    ArrayList<Chunk> chunks = new ArrayList<Chunk>();
+    chunks.add(new ChunkImpl("Hadoop", "namenode", 
+        System.currentTimeMillis(), "This is a test.".getBytes(), null));      
+    try {
+      QueryResponse rsp = server.query(new SolrQuery("*:*"));
+      Assert.assertEquals(0, rsp.getResults().getNumFound());
+
+      SolrWriter sw = new SolrWriter();
+      sw.add(chunks);
+      
+      // TODO: not a great way to test this - timing is easily out
+      // of whack due to parallel tests and various computer specs/load
+      Thread.sleep(1000); // wait 1 sec
+      
+      // now check that it comes out...
+      rsp = server.query(new SolrQuery("data:test"));
+      
+      int cnt = 0;
+      while (rsp.getResults().getNumFound() == 0) {
+        // wait and try again for slower/busier machines
+        // and/or parallel test effects.
+        
+        if (cnt++ == 10) {
+          break;
+        }
+        
+        Thread.sleep(2000); // wait 2 seconds...
+        
+        rsp = server.query(new SolrQuery("data:test"));
+      }
+      
+      Assert.assertEquals(1, rsp.getResults().getNumFound());
+      
+    } catch (Exception e) {
+      
+    }
+  }
+}



Mime
View raw message