metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject [06/21] metron git commit: METRON-1776 Update public web site to point at 0.6.0 new release (justinleet) closes apache/metron#1195
Date Tue, 18 Sep 2018 14:54:47 GMT
http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-pcap-backend/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-platform/metron-pcap-backend/index.html b/site/current-book/metron-platform/metron-pcap-backend/index.html
index 5206fd0..e1c3000 100644
--- a/site/current-book/metron-platform/metron-pcap-backend/index.html
+++ b/site/current-book/metron-platform/metron-pcap-backend/index.html
@@ -1,13 +1,13 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-pcap-backend/index.md at 2018-06-07
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-pcap-backend/index.md at 2018-09-12
  | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Metron PCAP Backend</title>
     <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
       <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
       <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
     <li class="active ">Metron PCAP Backend</li>
-        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
-          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
         </ul>
       </div>
       <div class="row-fluid">
@@ -55,15 +55,16 @@
     <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
     <ul class="nav nav-list">
     <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
-    <li><a href="../../metron-platform/metron-api/index.html" title="Api"><span class="none"></span>Api</a></li>
     <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
     <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
     <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
     <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
     <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+    <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
     <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
     <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
     <li class="active"><a href="#"><span class="none"></span>Pcap-backend</a></li>
+    <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
     <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
     </ul>
 </li>
@@ -227,12 +228,14 @@ limitations under the License.
  -nr,--num_reducers &lt;arg&gt;        The number of reducers to use.  Default
                                  is 10.
  -h,--help                       Display help
+ -ps,--print_status              Print the status of the job as it runs
  -ir,--include_reverse           Indicates if filter should check swapped
                                  src/dest addresses and IPs
  -p,--protocol &lt;arg&gt;             IP Protocol
  -sa,--ip_src_addr &lt;arg&gt;         Source IP address
  -sp,--ip_src_port &lt;arg&gt;         Source port
  -st,--start_time &lt;arg&gt;          (required) Packet start time range.
+ -yq,--yarn_queue &lt;arg&gt;          Yarn queue this job will be submitted to
 </pre></div></div>
 
 <div>
@@ -250,8 +253,10 @@ limitations under the License.
  -nr,--num_reducers &lt;arg&gt;        The number of reducers to use.  Default
                                  is 10.
  -h,--help                       Display help
+ -ps,--print_status              Print the status of the job as it runs
  -q,--query &lt;arg&gt;                Query string to use as a filter
  -st,--start_time &lt;arg&gt;          (required) Packet start time range.
+ -yq,--yarn_queue &lt;arg&gt;          Yarn queue this job will be submitted to
 </pre></div></div>
 
 <p>The Query filter&#x2019;s <tt>--query</tt> argument specifies the Stellar expression to execute on each packet.  To interact with the packet, a few variables are exposed:</p>

http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-solr/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-platform/metron-solr/index.html b/site/current-book/metron-platform/metron-solr/index.html
new file mode 100644
index 0000000..1afb245
--- /dev/null
+++ b/site/current-book/metron-platform/metron-solr/index.html
@@ -0,0 +1,300 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-solr/index.md at 2018-09-12
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
+    <meta http-equiv="Content-Language" content="en" />
+    <title>Metron &#x2013; Solr in Metron</title>
+    <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
+    <link rel="stylesheet" href="../../css/site.css" />
+    <link rel="stylesheet" href="../../css/print.css" media="print" />
+    <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+              $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+            </script>
+  </head>
+  <body class="topBarDisabled">
+    <div class="container-fluid">
+      <div id="banner">
+        <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../images/metron-logo.png"  alt="Apache Metron" width="148px" height="48px"/></a></div>
+        <div class="pull-right"></div>
+        <div class="clear"><hr/></div>
+      </div>
+
+      <div id="breadcrumbs">
+        <ul class="breadcrumb">
+      <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+      <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+      <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+    <li class="active ">Solr in Metron</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
+        </ul>
+      </div>
+      <div class="row-fluid">
+        <div id="leftColumn" class="span2">
+          <div class="well sidebar-nav">
+    <ul class="nav nav-list">
+      <li class="nav-header">User Documentation</li>
+    <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+    <ul class="nav nav-list">
+    <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+    <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+    <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+    <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+    <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+    <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+    <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
+    <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
+    <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+    <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+    <ul class="nav nav-list">
+    <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+    <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+    <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+    <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+    <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+    <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+    <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
+    <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+    <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+    <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+    <li class="active"><a href="#"><span class="none"></span>Solr</a></li>
+    <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+    </ul>
+</li>
+    <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+    <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+    <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+    <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+    <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+    </ul>
+</li>
+</ul>
+          <hr />
+          <div id="poweredBy">
+            <div class="clear"></div>
+            <div class="clear"></div>
+            <div class="clear"></div>
+            <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a>
+            </div>
+          </div>
+        </div>
+        <div id="bodyColumn"  class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Solr in Metron</h1>
+<p><a name="Solr_in_Metron"></a></p>
+<div class="section">
+<h2><a name="Table_of_Contents"></a>Table of Contents</h2>
+<ul>
+
+<li><a href="#Introduction">Introduction</a></li>
+<li><a href="#Configuration">Configuration</a></li>
+<li><a href="#Installing">Installing</a></li>
+<li><a href="#Schemas">Schemas</a></li>
+<li><a href="#Collections">Collections</a></li>
+</ul></div>
+<div class="section">
+<h2><a name="Introduction"></a>Introduction</h2>
+<p>Metron ships with Solr 6.6.2 support. Solr Cloud can be used as the real-time portion of the datastore resulting from <a href="../metron-indexing/index.html">metron-indexing</a>.</p></div>
+<div class="section">
+<h2><a name="Configuration"></a>Configuration</h2>
+<div class="section">
+<h3><a name="The_Indexing_Topology"></a>The Indexing Topology</h3>
+<p>Solr is a viable option for the <tt>random access topology</tt> and, similar to the Elasticsearch Writer, can be configured via the global config.  The following settings are possible as part of the global config:</p>
+<ul>
+
+<li><tt>solr.zookeeper</tt>
+<ul>
+
+<li>The zookeeper quorum associated with the SolrCloud instance.  This is a required field with no default.</li>
+</ul>
+</li>
+<li><tt>solr.commitPerBatch</tt>
+<ul>
+
+<li>This is a boolean which defines whether the writer commits every batch.  The default is <tt>true</tt>.</li>
+<li><i>WARNING</i>: If you set this to <tt>false</tt>, then commits will happen based on the SolrClient&#x2019;s internal mechanism and worker failure <i>may</i> result data being acknowledged in storm but not written in Solr.</li>
+</ul>
+</li>
+<li><tt>solr.commit.soft</tt>
+<ul>
+
+<li>This is a boolean which defines whether the writer makes a soft commit or a durable commit.  See <a class="externalLink" href="https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits">here</a>  The default is <tt>false</tt>.</li>
+<li><i>WARNING</i>: If you set this to <tt>true</tt>, then commits will happen based on the SolrClient&#x2019;s internal mechanism and worker failure <i>may</i> result data being acknowledged in storm but not written in Solr.</li>
+</ul>
+</li>
+<li><tt>solr.commit.waitSearcher</tt>
+<ul>
+
+<li>This is a boolean which defines whether the writer blocks the commit until the data is available to search.  See <a class="externalLink" href="https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits">here</a>  The default is <tt>true</tt>.</li>
+<li><i>WARNING</i>: If you set this to <tt>false</tt>, then commits will happen based on the SolrClient&#x2019;s internal mechanism and worker failure <i>may</i> result data being acknowledged in storm but not written in Solr.</li>
+</ul>
+</li>
+<li><tt>solr.commit.waitFlush</tt>
+<ul>
+
+<li>This is a boolean which defines whether the writer blocks the commit until the data is flushed.  See <a class="externalLink" href="https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits">here</a>  The default is <tt>true</tt>.</li>
+<li><i>WARNING</i>: If you set this to <tt>false</tt>, then commits will happen based on the SolrClient&#x2019;s internal mechanism and worker failure <i>may</i> result data being acknowledged in storm but not written in Solr.</li>
+</ul>
+</li>
+<li><tt>solr.collection</tt>
+<ul>
+
+<li>The default solr collection (if unspecified, the name is <tt>metron</tt>).  By default, sensors will write to a collection associated with the index name in the indexing config for that sensor.  If that index name is the empty string, then the default collection will be used.</li>
+</ul>
+</li>
+<li><tt>solr.http.config</tt>
+<ul>
+
+<li>This is a map which allows users to configure the Solr client&#x2019;s HTTP client.</li>
+<li>Possible fields here are:
+<ul>
+
+<li><tt>socketTimeout</tt> : Socket timeout measured in ms, closes a socket if read takes longer than x ms to complete throws <tt>java.net.SocketTimeoutException: Read timed out exception</tt></li>
+<li><tt>connTimeout</tt> : Connection timeout measures in ms, closes a socket if connection cannot be established within x ms with a <tt>java.net.SocketTimeoutException: Connection timed out</tt></li>
+<li><tt>maxConectionsPerHost</tt> : Maximum connections allowed per host</li>
+<li><tt>maxConnections</tt> :  Maximum total connections allowed</li>
+<li><tt>retry</tt> : Retry http requests on error</li>
+<li><tt>allowCompression</tt> :  Allow compression (deflate,gzip) if server supports it</li>
+<li><tt>followRedirects</tt> : Follow redirects</li>
+<li><tt>httpBasicAuthUser</tt> : Basic auth username</li>
+<li><tt>httpBasicAuthPassword</tt> : Basic auth password</li>
+<li><tt>solr.ssl.checkPeerName</tt> : Check peer name</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul></div></div>
+<div class="section">
+<h2><a name="Installing"></a>Installing</h2>
+<p>Solr is installed in the <a href="../../metron-deployment/development/centos6/index.html">full dev environment for CentOS</a> by default but is not started initially.  Navigate to <tt>$METRON_HOME/bin</tt> and start Solr Cloud by running <tt>start_solr.sh</tt>.</p>
+<p>Metron&#x2019;s Ambari MPack installs several scripts in <tt>$METRON_HOME/bin</tt> that can be used to manage Solr.  A script is also provided for installing Solr Cloud outside of full dev. The script performs the following tasks</p>
+<ul>
+
+<li>Stops ES and Kibana</li>
+<li>Downloads Solr</li>
+<li>Installs Solr</li>
+<li>Starts Solr Cloud</li>
+</ul>
+<p><i>Note: for details on setting up Solr Cloud in production mode, see <a class="externalLink" href="https://lucene.apache.org/solr/guide/6_6/taking-solr-to-production.html">https://lucene.apache.org/solr/guide/6_6/taking-solr-to-production.html</a></i></p>
+<p>Navigate to <tt>$METRON_HOME/bin</tt> and spin up Solr Cloud by running <tt>install_solr.sh</tt>.  After running this script, Elasticsearch and Kibana will have been stopped and you should now have an instance of Solr Cloud up and running at <a class="externalLink" href="http://localhost:8983/solr/#/~cloud">http://localhost:8983/solr/#/~cloud</a>.  This manner of starting Solr will also spin up an embedded Zookeeper instance at port 9983. More information can be found <a class="externalLink" href="https://lucene.apache.org/solr/guide/6_6/getting-started-with-solrcloud.html">here</a></p>
+<p>Solr can also be installed using <a class="externalLink" href="https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.6.4/bk_solr-search-installation/content/ch_hdp_search_30.html">HDP Search 3</a>.  HDP Search 3 sets the Zookeeper root to <tt>/solr</tt> so this will need to be added to each url in the comma-separated list in Ambari UI -&gt; Services -&gt; Metron -&gt; Configs -&gt; Index Settings -&gt; Solr Zookeeper Urls.  For example, in full dev this would be <tt>node1:2181/solr</tt>.</p></div>
+<div class="section">
+<h2><a name="Enabling_Solr"></a>Enabling Solr</h2>
+<p>Elasticsearch is the real-time store used by default in Metron.  Solr can be enabled following these steps:</p>
+<ol style="list-style-type: decimal">
+
+<li>Stop the Metron Indexing component in Ambari.</li>
+<li>Update Ambari UI -&gt; Services -&gt; Metron -&gt; Configs -&gt; Index Settings -&gt; Solr Zookeeper Urls to match the Solr installation described in the previous section.</li>
+<li>Change Ambari UI -&gt; Services -&gt; Metron -&gt; Configs -&gt; Indexing -&gt; Index Writer - Random Access -&gt; Random Access Search Engine to <tt>Solr</tt>.</li>
+<li>Set the <tt>source.type.field</tt> property to <tt>source.type</tt> in the <a href="../metron-common/index.html#Global_Configuration">Global Configuration</a>.</li>
+<li>Set the <tt>threat.triage.score.field</tt> property to <tt>threat.triage.score</tt> in the <a href="../metron-common/index.html#Global_Configuration">Global Configuration</a>.</li>
+<li>Start the Metron Indexing component in Ambari.</li>
+<li>Restart Metron REST and the Alerts UI in Ambari.</li>
+</ol>
+<p>This will automatically create collections for the schemas shipped with Metron:</p>
+<ul>
+
+<li>bro</li>
+<li>snort</li>
+<li>yaf</li>
+<li>error (used internally by Metron)</li>
+<li>metaalert (used internall by Metron)</li>
+</ul>
+<p>Any other collections must be created manually before starting the Indexing component.  Alerts should be present in the Alerts UI after enabling Solr.</p></div>
+<div class="section">
+<h2><a name="Schemas"></a>Schemas</h2>
+<p>As of now, we have mapped out the Schemas in <tt>src/main/config/schema</tt>. Ambari will eventually install these, but at the moment it&#x2019;s manual and you should refer to the Solr documentation <a href="here/index.html">https://lucene.apache.org/solr/guide/6_6</a> in general and <a class="externalLink" href="https://lucene.apache.org/solr/guide/6_6/documents-fields-and-schema-design.html">here</a> if you&#x2019;d like to know more about schemas in Solr.</p>
+<p>In Metron&#x2019;s Solr DAO implementation, document updates involve reading a document, applying the update and replacing the original by reindexing the whole document.<br />
+Indexing LatLonType and PointType field types stores data in internal fields that should not be returned in search results.  For these fields a dynamic field type matching the suffix needs to be added to store the data points. Solr 6+ comes with a new LatLonPointSpatialField field type that should be used instead of LatLonType if possible.  Otherwise, a LatLongType field should be defined as:</p>
+
+<div>
+<div>
+<pre class="source">&lt;dynamicField name=&quot;*.location_point&quot; type=&quot;location&quot; multiValued=&quot;false&quot; docValues=&quot;false&quot;/&gt;
+&lt;dynamicField name=&quot;*_coordinate&quot; type=&quot;pdouble&quot; indexed=&quot;true&quot; stored=&quot;false&quot; docValues=&quot;false&quot;/&gt;
+&lt;fieldType name=&quot;location&quot; class=&quot;solr.LatLonType&quot; subFieldSuffix=&quot;_coordinate&quot;/&gt;
+</pre></div></div>
+
+<p>A PointType field should be defined as:</p>
+
+<div>
+<div>
+<pre class="source">&lt;dynamicField name=&quot;*.point&quot; type=&quot;point&quot; multiValued=&quot;false&quot; docValues=&quot;false&quot;/&gt;
+&lt;dynamicField name=&quot;*_point&quot; type=&quot;pdouble&quot; indexed=&quot;true&quot; stored=&quot;false&quot; docValues=&quot;false&quot;/&gt;
+&lt;fieldType name=&quot;point&quot; class=&quot;solr.PointType&quot; subFieldSuffix=&quot;_point&quot;/&gt;
+</pre></div></div>
+
+<p>If any copy fields are defined, stored and docValues should be set to false.</p></div>
+<div class="section">
+<h2><a name="Collections"></a>Collections</h2>
+<p>Convenience scripts are provided with Metron to create and delete collections.  Ambari uses these scripts to automatically create collections.  To use them outside of Ambari, a few environment variables must be set first:</p>
+
+<div>
+<div>
+<pre class="source"># Path to the zookeeper node used by Solr
+export ZOOKEEPER=node1:2181/solr
+# Set to true if Kerberos is enabled
+export SECURITY_ENABLED=true 
+</pre></div></div>
+
+<p>The scripts can then be called directly with the collection name as the first argument .  For example, to create the bro collection:</p>
+
+<div>
+<div>
+<pre class="source">$METRON_HOME/bin/create_collection.sh bro
+</pre></div></div>
+
+<p>To delete the bro collection:</p>
+
+<div>
+<div>
+<pre class="source">$METRON_HOME/bin/delete_collection.sh bro
+</pre></div></div>
+
+<p>The <tt>create_collection.sh</tt> script depends on schemas installed in <tt>$METRON_HOME/config/schema</tt>.  There are several schemas that come with Metron:</p>
+<ul>
+
+<li>bro</li>
+<li>snort</li>
+<li>yaf</li>
+<li>metaalert</li>
+<li>error</li>
+</ul>
+<p>Additional schemas should be installed in that location if using the <tt>create_collection.sh</tt> script.  Any collection can be deleted with the <tt>delete_collection.sh</tt> script. These scripts use the <a class="externalLink" href="http://lucene.apache.org/solr/guide/6_6/collections-api.html">Solr Collection API</a>.</p></div>
+        </div>
+      </div>
+    </div>
+    <hr/>
+    <footer>
+      <div class="container-fluid">
+        <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+            and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+        </div>
+      </div>
+    </footer>
+  </body>
+</html>

http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-writer/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-platform/metron-writer/index.html b/site/current-book/metron-platform/metron-writer/index.html
index cb1a26f..bee846f 100644
--- a/site/current-book/metron-platform/metron-writer/index.html
+++ b/site/current-book/metron-platform/metron-writer/index.html
@@ -1,13 +1,13 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-writer/index.md at 2018-06-07
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-writer/index.md at 2018-09-12
  | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Writer</title>
     <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
       <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
       <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
     <li class="active ">Writer</li>
-        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
-          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
         </ul>
       </div>
       <div class="row-fluid">
@@ -55,15 +55,16 @@
     <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
     <ul class="nav nav-list">
     <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
-    <li><a href="../../metron-platform/metron-api/index.html" title="Api"><span class="none"></span>Api</a></li>
     <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
     <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
     <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
     <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
     <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+    <li><a href="../../metron-platform/metron-job/index.html" title="Job"><span class="none"></span>Job</a></li>
     <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
     <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
     <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+    <li><a href="../../metron-platform/metron-solr/index.html" title="Solr"><span class="none"></span>Solr</a></li>
     <li class="active"><a href="#"><span class="none"></span>Writer</a></li>
     </ul>
 </li>
@@ -108,6 +109,28 @@ limitations under the License.
 <h2><a name="Introduction"></a>Introduction</h2>
 <p>The writer module provides some utilties for writing to outside components from within Storm.  This includes managing bulk writing.  An implemention is included for writing to HDFS in this module. Other writers can be found in their own modules.</p></div>
 <div class="section">
+<h2><a name="Kafka_Writer"></a>Kafka Writer</h2>
+<p>We have an implementation of a writer which will write batches of messages to Kafka.  An interesting aspect of this writer is that it can be configured to allow users to specify a message field which contains the topic for the message.</p>
+<p>The configuration for this writer is held in the individual Sensor Configurations:</p>
+<ul>
+
+<li><a href="../metron-enrichment/index.html#sensor-enrichment-configuration">Enrichment</a> under the <tt>config</tt> element</li>
+<li><a href="../metron-parsers/index.html#parser-configuration">Parsers</a> in the <tt>parserConfig</tt> element</li>
+<li>Profiler - Unsupported currently</li>
+</ul>
+<p>In each of these, the kafka writer can be configured via a map which has the following elements:</p>
+<ul>
+
+<li><tt>kafka.brokerUrl</tt> : The broker URL</li>
+<li><tt>kafka.keySerializer</tt> : The key serializer (defaults to <tt>StringSerializer</tt>)</li>
+<li><tt>kafka.valueSerializer</tt> : The key serializer (defaults to <tt>StringSerializer</tt>)</li>
+<li><tt>kafka.zkQuorum</tt> : The zookeeper quorum</li>
+<li><tt>kafka.requiredAcks</tt> : Whether to require acks.</li>
+<li><tt>kafka.topic</tt> : The topic to write to</li>
+<li><tt>kafka.topicField</tt> : The field to pull the topic from.  If this is specified, then the producer will use this.  If it is unspecified, then it will default to the <tt>kafka.topic</tt> property.  If neither are specified, then an error will occur.</li>
+<li><tt>kafka.producerConfigs</tt> : A map of kafka producer configs for advanced customization.</li>
+</ul></div>
+<div class="section">
 <h2><a name="HDFS_Writer"></a>HDFS Writer</h2>
 <p>The HDFS writer included here expands on what Storm has in several ways. There&#x2019;s customization in syncing to HDFS, rotation policy, etc. In addition, the writer allows for users to define output paths based on the fields in the provided JSON message.  This can be defined using Stellar.</p>
 <p>To manage the output path, a base path argument is provided by the Flux file, with the FileNameFormat as follows</p>

http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-sensors/fastcapa/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-sensors/fastcapa/index.html b/site/current-book/metron-sensors/fastcapa/index.html
index 1b23080..8aef549 100644
--- a/site/current-book/metron-sensors/fastcapa/index.html
+++ b/site/current-book/metron-sensors/fastcapa/index.html
@@ -1,13 +1,13 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-sensors/fastcapa/index.md at 2018-06-07
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-sensors/fastcapa/index.md at 2018-09-12
  | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Fastcapa</title>
     <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
       <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
       <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
     <li class="active ">Fastcapa</li>
-        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
-          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
         </ul>
       </div>
       <div class="row-fluid">

http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-sensors/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-sensors/index.html b/site/current-book/metron-sensors/index.html
index a40e51d..ea514b7 100644
--- a/site/current-book/metron-sensors/index.html
+++ b/site/current-book/metron-sensors/index.html
@@ -1,13 +1,13 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-sensors/index.md at 2018-06-07
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-sensors/index.md at 2018-09-12
  | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Metron Sensors</title>
     <link rel="stylesheet" href="../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
       <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
       <li class=""><a href="../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
     <li class="active ">Metron Sensors</li>
-        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
-          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
         </ul>
       </div>
       <div class="row-fluid">

http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-sensors/pycapa/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-sensors/pycapa/index.html b/site/current-book/metron-sensors/pycapa/index.html
index f81aee9..e7a507e 100644
--- a/site/current-book/metron-sensors/pycapa/index.html
+++ b/site/current-book/metron-sensors/pycapa/index.html
@@ -1,13 +1,13 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-sensors/pycapa/index.md at 2018-06-07
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-sensors/pycapa/index.md at 2018-09-12
  | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Pycapa</title>
     <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
       <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
       <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
     <li class="active ">Pycapa</li>
-        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
-          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
         </ul>
       </div>
       <div class="row-fluid">
@@ -99,7 +99,13 @@ limitations under the License.
 <ul>
 
 <li><a href="#Overview">Overview</a></li>
-<li><a href="#Installation">Installation</a></li>
+<li><a href="#Installation">Installation</a>
+<ul>
+
+<li><a href="#Centos_7">Centos 7</a></li>
+<li><a href="#Centos_6">Centos 6</a></li>
+</ul>
+</li>
 <li><a href="#Usage">Usage</a>
 <ul>
 
@@ -109,24 +115,30 @@ limitations under the License.
 </ul>
 </li>
 <li><a href="#FAQs">FAQs</a></li>
-</ul>
-<h1>Overview</h1>
-<p>Pycapa performs network packet capture, both off-the-wire and from a Kafka topic, which is useful for the testing and development of <a class="externalLink" href="https://github.com/apache/metron">Apache Metron</a>.  It is not intended for production use. The tool will capture packets from a specified interface and push them into a Kafka Topic.  The tool can also do the reverse.  It can consume packets from Kafka and reconstruct each network packet.  This can then be used to create a <a class="externalLink" href="https://wiki.wireshark.org/Development/LibpcapFileFormat">libpcap-compliant file</a> or even to feed directly into a tool like Wireshark to monitor ongoing activity.</p>
-<h1>Installation</h1>
+</ul></div>
+<div class="section">
+<h2><a name="Overview"></a>Overview</h2>
+<p>Pycapa performs network packet capture, both off-the-wire and from a Kafka topic, which is useful for the testing and development of <a class="externalLink" href="https://github.com/apache/metron">Apache Metron</a>.  It is not intended for production use. The tool will capture packets from a specified interface and push them into a Kafka Topic.  The tool can also do the reverse.  It can consume packets from Kafka and reconstruct each network packet.  This can then be used to create a <a class="externalLink" href="https://wiki.wireshark.org/Development/LibpcapFileFormat">libpcap-compliant file</a> or even to feed directly into a tool like Wireshark to monitor ongoing activity.</p></div>
+<div class="section">
+<h2><a name="Installation"></a>Installation</h2>
 <p>General notes on the installation of Pycapa.</p>
 <ul>
 
 <li>Python 2.7 is required.</li>
-<li>The following package dependencies are required and can be installed automatically with <tt>pip</tt>.
+<li>The following package dependencies are required and can be installed automatically with <tt>pip</tt>. The requirements are installed as part of step 4
 <ul>
 
 <li><a class="externalLink" href="https://github.com/confluentinc/confluent-kafka-python">confluent-kafka-python</a></li>
 <li><a class="externalLink" href="https://github.com/CoreSecurity/pcapy">pcapy</a></li>
 </ul>
 </li>
+</ul>
+<div class="section">
+<h3><a name="Centos_7"></a>Centos 7</h3>
+<ul>
+
 <li>These instructions can be used directly on CentOS 7+.</li>
 <li>Other Linux distributions that come with Python 2.7 can use these instructions with some minor modifications.</li>
-<li>Older distributions, like CentOS 6, that come with Python 2.6 installed, should install Python 2.7 within a virtual environment and then run Pycapa from within the virtual environment.</li>
 </ul>
 <ol style="list-style-type: decimal">
 
@@ -146,8 +158,8 @@ limitations under the License.
 <div>
 <div>
 <pre class="source">export PREFIX=/usr
-wget https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz   -O - | tar -xz
-cd librdkafka-0.9.4/
+wget https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz   -O - | tar -xz
+cd librdkafka-0.11.5/
 ./configure --prefix=$PREFIX
 make
 make install
@@ -174,8 +186,115 @@ pip install -r requirements.txt
 python setup.py install
 </pre></div></div>
 </li>
+</ol></div>
+<div class="section">
+<h3><a name="Centos_6"></a>Centos 6</h3>
+<ul>
+
+<li>These instructions can be used directly on CentOS 6 - useful for developers using the Full Dev Vagrant test box.</li>
+<li>Older distributions, like CentOS 6, that come with Python 2.6 installed, should install Python 2.7 within a virtual environment and then run Pycapa from within the virtual environment.</li>
+</ul>
+<ol style="list-style-type: decimal">
+
+<li>
+
+<p>Set up a couple environment variables.</p>
+
+<div>
+<div>
+<pre class="source">PYCAPA_HOME=/opt/pycapa
+PYTHON27_HOME=/opt/rh/python27/root
+</pre></div></div>
+</li>
+<li>
+
+<p>Install required packages.</p>
+
+<div>
+<div>
+<pre class="source">for item in epel-release centos-release-scl &quot;@Development tools&quot; python27 python27-scldevel python27-python-virtualenv libpcap-devel libselinux-python; do yum install -y $item; done
+</pre></div></div>
+</li>
+<li>
+
+<p>Setup Pycapa directory.</p>
+
+<div>
+<div>
+<pre class="source">mkdir $PYCAPA_HOME &amp;&amp; chmod 755 $PYCAPA_HOME
+</pre></div></div>
+</li>
+<li>
+
+<p>Create the virtualenv.</p>
+
+<div>
+<div>
+<pre class="source">export LD_LIBRARY_PATH=&quot;/opt/rh/python27/root/usr/lib64&quot;
+cd $PYCAPA_HOME
+${PYTHON27_HOME}/usr/bin/virtualenv pycapa-venv
+</pre></div></div>
+</li>
+<li>
+
+<p>Install Librdkafka at your chosen $PREFIX.</p>
+
+<div>
+<div>
+<pre class="source">export PREFIX=/usr
+wget https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz   -O - | tar -xz
+cd librdkafka-0.11.5/
+./configure --prefix=$PREFIX
+make
+make install
+</pre></div></div>
+</li>
+<li>
+
+<p>Add Librdkafka to the dynamic library load path.</p>
+
+<div>
+<div>
+<pre class="source">echo &quot;$PREFIX/lib&quot; &gt;&gt; /etc/ld.so.conf.d/pycapa.conf
+ldconfig -v
+</pre></div></div>
+</li>
+<li>
+
+<p>Copy the Pycapa source files from the Metron project to your chosen $PYCAPA_HOME (e.g. <tt>/opt/pycapa</tt>). You should have pycapa source files in <tt>/opt/pycapa/pycapa</tt>.</p>
+
+<div>
+<div>
+<pre class="source">scp -r metron-sensors/pycapa root@node1:$PYCAPA_HOME
+</pre></div></div>
+</li>
+<li>
+
+<p>Install Pycapa using the <tt>pycapa-venv</tt> virtualenv you created earlier.</p>
+
+<div>
+<div>
+<pre class="source">cd ${PYCAPA_HOME}/pycapa
+# activate the virtualenv
+source ${PYCAPA_HOME}/pycapa-venv/bin/activate
+pip install -r requirements.txt
+python setup.py install
+</pre></div></div>
+</li>
+<li>
+
+<p>Special notes on running pycapa on Centos 6. You should run it using the virtualenv.</p>
+
+<div>
+<div>
+<pre class="source">cd ${PYCAPA_HOME}/pycapa-venv/bin
+pycapa --producer --kafka-topic pcap --interface eth1 --kafka-broker $BROKERLIST
+</pre></div></div>
+</li>
 </ol>
-<h1>Usage</h1>
+<p><b>Note:</b> To deactivate your virtualenv, simply type &#x201c;deactivate&#x201d; and hit enter.</p></div></div>
+<div class="section">
+<h2><a name="Usage"></a>Usage</h2>
 <p>Pycapa has two primary runtime modes.</p>
 <ul>
 
@@ -341,7 +460,7 @@ Capturing on 'Standard input'
 </div></div>
 <div class="section">
 <h3><a name="Kerberos"></a>Kerberos</h3>
-<p>The probe can be used in a Kerberized environment.  Follow these additional steps to use Pycapa with Kerberos.  The following assumptions have been made.  These may need altered to fit your environment.</p>
+<p>The probe can be used in a Kerberized environment. The Python client README (<a class="externalLink" href="https://github.com/confluentinc/confluent-kafka-python">https://github.com/confluentinc/confluent-kafka-python</a>) has an important note for Kerberos case that the pre-built Linux wheels do NOT contain SASL Kerberos support. You will need to use the non-binary wheel to install confluent-kafka-python and build/install librdkafka separately. Follow these additional steps to use Pycapa with Kerberos.  The following assumptions have been made.  These may need altered to fit your environment.</p>
 <ul>
 
 <li>The Kafka broker is at <tt>kafka1:6667</tt></li>
@@ -354,12 +473,21 @@ Capturing on 'Standard input'
 
 <li>
 
+<p>If it is not, ensure that you have <tt>libsasl</tt> or <tt>libsasl2</tt> installed.  On CentOS, this can be installed with the following command.</p>
+
+<div>
+<div>
+<pre class="source">    yum install -y cyrus-sasl cyrus-sasl-devel cyrus-sasl-gssapi
+</pre></div></div>
+</li>
+<li>
+
 <p>Build Librdkafka with SASL support (<tt>--enable-sasl</tt>) and install at your chosen $PREFIX.</p>
 
 <div>
 <div>
-<pre class="source">wget https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz  -O - | tar -xz
-cd librdkafka-0.9.4/
+<pre class="source">wget https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz  -O - | tar -xz
+cd librdkafka-0.11.5/
 ./configure --prefix=$PREFIX --enable-sasl
 make
 make install
@@ -371,15 +499,21 @@ make install
 
 <div>
 <div>
-<pre class="source">$ examples/rdkafka_example -X builtin.features
-builtin.features = gzip,snappy,ssl,sasl,regex
+<pre class="source">$ examples/rdkafka_example -X builtin.features    
+  builtin.features = gzip,snappy,ssl,sasl,regex,lz4,sasl_gssapi,sasl_plain,sasl_scram,plugins
 </pre></div></div>
+</li>
+<li>The source install of confluent-kafka.
+<p>If you have already installed, remove the binary wheel python client first, repeat until it says no longer installed</p>
 
-<p>If it is not, ensure that you have <tt>libsasl</tt> or <tt>libsasl2</tt> installed.  On CentOS, this can be installed with the following command.</p>
+<div>
+<div>
+<pre class="source">   pip uninstall -y confluent-kafka 
+</pre></div></div>
 
 <div>
 <div>
-<pre class="source">yum install -y cyrus-sasl cyrus-sasl-devel cyrus-sasl-gssapi
+<pre class="source">   pip install --no-binary :all: confluent-kafka
 </pre></div></div>
 </li>
 <li>
@@ -428,8 +562,9 @@ INFO:root:'10' packet(s) in, '10' packet(s) out
 </li>
 </ul>
 </li>
-</ol>
-<h1>FAQs</h1></div>
+</ol></div></div>
+<div class="section">
+<h2><a name="FAQs"></a>FAQs</h2>
 <div class="section">
 <h3><a name="How_do_I_get_more_logs.3F"></a>How do I get more logs?</h3>
 <p>Use the following two command-line arguments to get detailed logging.</p>

http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-stellar/stellar-3rd-party-example/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-stellar/stellar-3rd-party-example/index.html b/site/current-book/metron-stellar/stellar-3rd-party-example/index.html
index 4e19e16..c04add7 100644
--- a/site/current-book/metron-stellar/stellar-3rd-party-example/index.html
+++ b/site/current-book/metron-stellar/stellar-3rd-party-example/index.html
@@ -1,13 +1,13 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-3rd-party-example/index.md at 2018-06-07
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-3rd-party-example/index.md at 2018-09-12
  | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Introduction</title>
     <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
       <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
       <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
     <li class="active ">Introduction</li>
-        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
-          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
         </ul>
       </div>
       <div class="row-fluid">

http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-stellar/stellar-common/3rdPartyStellar.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-stellar/stellar-common/3rdPartyStellar.html b/site/current-book/metron-stellar/stellar-common/3rdPartyStellar.html
index 88d7ac6..157d632 100644
--- a/site/current-book/metron-stellar/stellar-common/3rdPartyStellar.html
+++ b/site/current-book/metron-stellar/stellar-common/3rdPartyStellar.html
@@ -1,13 +1,13 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-common/3rdPartyStellar.md at 2018-06-07
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-common/3rdPartyStellar.md at 2018-09-12
  | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Custom Stellar Functions</title>
     <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
       <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
       <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
     <li class="active ">Custom Stellar Functions</li>
-        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
-          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
         </ul>
       </div>
       <div class="row-fluid">

http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-stellar/stellar-common/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-stellar/stellar-common/index.html b/site/current-book/metron-stellar/stellar-common/index.html
index 1160156..8ead4c6 100644
--- a/site/current-book/metron-stellar/stellar-common/index.html
+++ b/site/current-book/metron-stellar/stellar-common/index.html
@@ -1,13 +1,13 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-common/index.md at 2018-06-07
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-common/index.md at 2018-09-12
  | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Stellar Language</title>
     <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
       <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
       <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
     <li class="active ">Stellar Language</li>
-        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
-          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
         </ul>
       </div>
       <div class="row-fluid">
@@ -120,6 +120,7 @@ limitations under the License.
 <ul>
 
 <li>Referencing fields in the enriched JSON</li>
+<li>Referencing all fields in the enriched JSON via the <tt>_</tt> reserved variable name.</li>
 <li>String literals are quoted with either <tt>'</tt> or <tt>&quot;</tt></li>
 <li>String literals support escaping for <tt>'</tt>, <tt>&quot;</tt>, <tt>\t</tt>, <tt>\r</tt>, <tt>\n</tt>, and backslash
 <ul>
@@ -141,6 +142,16 @@ limitations under the License.
 <li>User defined functions, including Lambda expressions</li>
 </ul>
 <div class="section">
+<h3><a name="Boolean_Expressions"></a>Boolean Expressions</h3>
+<p>Variables may be used in boolean expressions and variables which are not explicitly boolean may be interpreted as booleans subject to the following rules:</p>
+<ul>
+
+<li>Similar to python and javascript, empty collections (e.g. <tt>[]</tt>) will be interpreted as <tt>false</tt></li>
+<li>Similar to python and javascript, missing variables will be interpreted as <tt>false</tt></li>
+<li>Variables set to <tt>null</tt> will be interpreted as <tt>false</tt></li>
+</ul>
+<p>Otherwise, boolean variables will be interpreted as their values reflect.</p></div>
+<div class="section">
 <h3><a name="Stellar_Language_Keywords"></a>Stellar Language Keywords</h3>
 <p>The following keywords need to be single quote escaped in order to be used in Stellar expressions:</p>
 <table border="0" class="table table-striped">
@@ -1772,12 +1783,12 @@ limitations under the License.
 <h3><a name="REGEXP_MATCH"></a><tt>REGEXP_MATCH</tt></h3>
 <ul>
 
-<li>Description: Determines whether a regex matches a string</li>
+<li>Description: Determines whether a regex matches a string.  If a list of patterns is passed, then the matching is an OR operation</li>
 <li>Input:
 <ul>
 
 <li>string - The string to test</li>
-<li>pattern - The proposed regex pattern</li>
+<li>pattern - The proposed regex pattern or a list of patterns</li>
 </ul>
 </li>
 <li>Returns: True if the regex pattern matches the string and false if otherwise.</li>
@@ -2674,7 +2685,7 @@ ABS, APPEND_IF_MISSING, BIN, BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, C
 
 <div>
 <div>
-<pre class="source">metron-stellar/stellar-common/target/stellar-common-0.5.0-stand-alone.tar.gz
+<pre class="source">metron-stellar/stellar-common/target/stellar-common-0.6.0-stand-alone.tar.gz
 </pre></div></div>
 
 <p>When unpacked, the following structure will be created:</p>
@@ -2685,7 +2696,7 @@ ABS, APPEND_IF_MISSING, BIN, BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, C
 &#x251c;&#x2500;&#x2500; bin
 &#x2502;&#xa0;&#xa0; &#x2514;&#x2500;&#x2500; stellar
 &#x2514;&#x2500;&#x2500; lib
-    &#x2514;&#x2500;&#x2500; stellar-common-0.5.0-uber.jar
+    &#x2514;&#x2500;&#x2500; stellar-common-0.6.0-uber.jar
 </pre></div></div>
 
 <p>To run the Stellar Shell run the following from the directory you unpacked to:</p>

http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-stellar/stellar-zeppelin/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-stellar/stellar-zeppelin/index.html b/site/current-book/metron-stellar/stellar-zeppelin/index.html
index 54bc800..128edba 100644
--- a/site/current-book/metron-stellar/stellar-zeppelin/index.html
+++ b/site/current-book/metron-stellar/stellar-zeppelin/index.html
@@ -1,13 +1,13 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-zeppelin/index.md at 2018-06-07
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-stellar/stellar-zeppelin/index.md at 2018-09-12
  | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Stellar Interpreter for Apache Zeppelin</title>
     <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
       <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
       <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
     <li class="active ">Stellar Interpreter for Apache Zeppelin</li>
-        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
-          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
         </ul>
       </div>
       <div class="row-fluid">
@@ -132,11 +132,11 @@ mvn clean install -DskipTests
 <li>
 
 <p>Use Zeppelin&#x2019;s installation utility to install the Stellar Interpreter.</p>
-<p>If Zeppelin was already installed, make sure that it is stopped before running this command.  Update the version, &#x2018;0.5.0&#x2019; in the example below, to whatever is appropriate for your environment.</p>
+<p>If Zeppelin was already installed, make sure that it is stopped before running this command.  Update the version, &#x2018;0.6.0&#x2019; in the example below, to whatever is appropriate for your environment.</p>
 
 <div>
 <div>
-<pre class="source">bin/install-interpreter.sh --name stellar --artifact org.apache.metron:stellar-zeppelin:0.5.0
+<pre class="source">bin/install-interpreter.sh --name stellar --artifact org.apache.metron:stellar-zeppelin:0.6.0
 </pre></div></div>
 </li>
 <li>
@@ -208,7 +208,7 @@ mvn clean install -DskipTests
 
 <div>
 <div>
-<pre class="source">org.apache.metron:metron-statistics:0.5.0
+<pre class="source">org.apache.metron:metron-statistics:0.6.0
 </pre></div></div>
 </li>
 <li>

http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/use-cases/forensic_clustering/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/use-cases/forensic_clustering/index.html b/site/current-book/use-cases/forensic_clustering/index.html
index f67a830..916a6c9 100644
--- a/site/current-book/use-cases/forensic_clustering/index.html
+++ b/site/current-book/use-cases/forensic_clustering/index.html
@@ -1,13 +1,13 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/forensic_clustering/index.md at 2018-06-07
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/forensic_clustering/index.md at 2018-09-12
  | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Problem Statement</title>
     <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
       <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
       <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
     <li class="active ">Problem Statement</li>
-        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
-          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
         </ul>
       </div>
       <div class="row-fluid">
@@ -61,6 +61,7 @@
     <ul class="nav nav-list">
     <li class="active"><a href="#"><span class="none"></span>Forensic_clustering</a></li>
     <li><a href="../../use-cases/geographic_login_outliers/index.html" title="Geographic_login_outliers"><span class="none"></span>Geographic_login_outliers</a></li>
+    <li><a href="../../use-cases/parser_chaining/index.html" title="Parser_chaining"><span class="none"></span>Parser_chaining</a></li>
     <li><a href="../../use-cases/typosquat_detection/index.html" title="Typosquat_detection"><span class="none"></span>Typosquat_detection</a></li>
     </ul>
 </li>
@@ -226,30 +227,140 @@ tar xzvf ~/180424243034750.tar.gz
 
 </pre></div></div>
 
-<p>Before we start, we will want to install ES mappings so ES knows how to interpret our fields:</p>
+<p>Before we start, we will want to install ES template mappings so ES knows how to interpret our fields:</p>
 
 <div>
 <div>
-<pre class="source">curl -XPUT 'http://$ES_HOST/cowrie*/_mapping/cowrie_doc' -d '
+<pre class="source">curl -XPUT $ES_HOST'/_template/cowrie_index' -d '
 {
+  &quot;template&quot;: &quot;cowrie_index*&quot;,
+  &quot;mappings&quot;: {
+    &quot;cowrie_doc&quot;: {
+        &quot;dynamic_templates&quot;: [
+        {
+          &quot;geo_location_point&quot;: {
+            &quot;match&quot;: &quot;enrichments:geo:*:location_point&quot;,
+            &quot;match_mapping_type&quot;: &quot;*&quot;,
+            &quot;mapping&quot;: {
+              &quot;type&quot;: &quot;geo_point&quot;
+            }
+          }
+        },
+        {
+          &quot;geo_country&quot;: {
+            &quot;match&quot;: &quot;enrichments:geo:*:country&quot;,
+            &quot;match_mapping_type&quot;: &quot;*&quot;,
+            &quot;mapping&quot;: {
+              &quot;type&quot;: &quot;keyword&quot;
+            }
+          }
+        },
+        {
+          &quot;geo_city&quot;: {
+            &quot;match&quot;: &quot;enrichments:geo:*:city&quot;,
+            &quot;match_mapping_type&quot;: &quot;*&quot;,
+            &quot;mapping&quot;: {
+              &quot;type&quot;: &quot;keyword&quot;
+            }
+          }
+        },
+        {
+          &quot;geo_location_id&quot;: {
+            &quot;match&quot;: &quot;enrichments:geo:*:locID&quot;,
+            &quot;match_mapping_type&quot;: &quot;*&quot;,
+            &quot;mapping&quot;: {
+              &quot;type&quot;: &quot;keyword&quot;
+            }
+          }
+        },
+        {
+          &quot;geo_dma_code&quot;: {
+            &quot;match&quot;: &quot;enrichments:geo:*:dmaCode&quot;,
+            &quot;match_mapping_type&quot;: &quot;*&quot;,
+            &quot;mapping&quot;: {
+              &quot;type&quot;: &quot;keyword&quot;
+            }
+          }
+        },
+        {
+          &quot;geo_postal_code&quot;: {
+            &quot;match&quot;: &quot;enrichments:geo:*:postalCode&quot;,
+            &quot;match_mapping_type&quot;: &quot;*&quot;,
+            &quot;mapping&quot;: {
+              &quot;type&quot;: &quot;keyword&quot;
+            }
+          }
+        },
+        {
+          &quot;geo_latitude&quot;: {
+            &quot;match&quot;: &quot;enrichments:geo:*:latitude&quot;,
+            &quot;match_mapping_type&quot;: &quot;*&quot;,
+            &quot;mapping&quot;: {
+              &quot;type&quot;: &quot;float&quot;
+            }
+          }
+        },
+        {
+          &quot;geo_longitude&quot;: {
+            &quot;match&quot;: &quot;enrichments:geo:*:longitude&quot;,
+            &quot;match_mapping_type&quot;: &quot;*&quot;,
+            &quot;mapping&quot;: {
+              &quot;type&quot;: &quot;float&quot;
+            }
+          }
+        },
+        {
+          &quot;timestamps&quot;: {
+            &quot;match&quot;: &quot;*:ts&quot;,
+            &quot;match_mapping_type&quot;: &quot;*&quot;,
+            &quot;mapping&quot;: {
+              &quot;type&quot;: &quot;date&quot;,
+              &quot;format&quot;: &quot;epoch_millis&quot;
+            }
+          }
+        },
+        {
+          &quot;threat_triage_score&quot;: {
+            &quot;mapping&quot;: {
+              &quot;type&quot;: &quot;float&quot;
+            },
+            &quot;match&quot;: &quot;threat:triage:*score&quot;,
+            &quot;match_mapping_type&quot;: &quot;*&quot;
+          }
+        },
+        {
+          &quot;threat_triage_reason&quot;: {
+            &quot;mapping&quot;: {
+              &quot;type&quot;: &quot;text&quot;,
+              &quot;fielddata&quot;: &quot;true&quot;
+            },
+            &quot;match&quot;: &quot;threat:triage:rules:*:reason&quot;,
+            &quot;match_mapping_type&quot;: &quot;*&quot;
+          }
+        },
+        {
+          &quot;threat_triage_name&quot;: {
+            &quot;mapping&quot;: {
+              &quot;type&quot;: &quot;text&quot;,
+              &quot;fielddata&quot;: &quot;true&quot;
+            },
+            &quot;match&quot;: &quot;threat:triage:rules:*:name&quot;,
+            &quot;match_mapping_type&quot;: &quot;*&quot;
+          }
+        }
+        ],
         &quot;properties&quot; : {
-          &quot;adapter:stellaradapter:begin:ts&quot; : {
-            &quot;type&quot; : &quot;string&quot;
-          },
-          &quot;adapter:stellaradapter:end:ts&quot; : {
-            &quot;type&quot; : &quot;string&quot;
-          },
           &quot;blacklisted&quot; : {
             &quot;type&quot; : &quot;boolean&quot;
           },
           &quot;compCS&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;data&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;dst_ip&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;dst_port&quot; : {
             &quot;type&quot; : &quot;long&quot;
@@ -258,117 +369,87 @@ tar xzvf ~/180424243034750.tar.gz
             &quot;type&quot; : &quot;double&quot;
           },
           &quot;encCS&quot; : {
-            &quot;type&quot; : &quot;string&quot;
-          },
-          &quot;enrichmentjoinbolt:joiner:ts&quot; : {
-            &quot;type&quot; : &quot;string&quot;
-          },
-          &quot;enrichmentsplitterbolt:splitter:begin:ts&quot; : {
-            &quot;type&quot; : &quot;string&quot;
-          },
-          &quot;enrichmentsplitterbolt:splitter:end:ts&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;eventid&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;guid&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;input&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;isError&quot; : {
             &quot;type&quot; : &quot;long&quot;
           },
           &quot;is_alert&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;kexAlgs&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;keyAlgs&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;macCS&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;message&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
-          &quot;original_string&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+          &quot;original_keyword&quot; : {
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;password&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;sensor&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;session&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;similarity_bin&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;size&quot; : {
             &quot;type&quot; : &quot;long&quot;
           },
           &quot;source:type&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;src_ip&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;src_port&quot; : {
             &quot;type&quot; : &quot;long&quot;
           },
           &quot;system&quot; : {
-            &quot;type&quot; : &quot;string&quot;
-          },
-          &quot;threat:triage:rules:0:comment&quot; : {
-            &quot;type&quot; : &quot;string&quot;
-          },
-          &quot;threat:triage:rules:0:name&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
-          &quot;threat:triage:rules:0:reason&quot; : {
-            &quot;type&quot; : &quot;string&quot;
-          },
-          &quot;threat:triage:rules:0:score&quot; : {
-            &quot;type&quot; : &quot;long&quot;
-          },
-          &quot;threat:triage:score&quot; : {
-            &quot;type&quot; : &quot;double&quot;
-          },
-          &quot;threatinteljoinbolt:joiner:ts&quot; : {
-            &quot;type&quot; : &quot;string&quot;
-          },
-          &quot;threatintelsplitterbolt:splitter:begin:ts&quot; : {
-            &quot;type&quot; : &quot;string&quot;
-          },
-          &quot;threatintelsplitterbolt:splitter:end:ts&quot; : {
-            &quot;type&quot; : &quot;string&quot;
-          },
-          &quot;timestamp&quot; : {
-            &quot;type&quot; : &quot;long&quot;
+          &quot;timestamp&quot;: {
+            &quot;type&quot;: &quot;date&quot;,
+            &quot;format&quot;: &quot;epoch_millis&quot;
           },
           &quot;tlsh&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;ttylog&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;username&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
           &quot;version&quot; : {
-            &quot;type&quot; : &quot;string&quot;
+            &quot;type&quot; : &quot;keyword&quot;
           },
-          &quot;alert&quot; : {
+          &quot;metron_alert&quot; : {
             &quot;type&quot; : &quot;nested&quot;
           }
         }
+     }
+  }
 }
 '
 </pre></div></div>
@@ -474,7 +555,7 @@ tar xzvf ~/180424243034750.tar.gz
       &quot;stellar&quot; : {
         &quot;config&quot; : [
           &quot;blacklisted := ENRICHMENT_EXISTS( 'blacklist', src_ip, 'threatintel', 't')&quot;,
-          &quot;is_alert := (exists(is_alert) &amp;&amp; is_alert) || blacklisted&quot;
+          &quot;is_alert := is_alert || blacklisted&quot;
         ]
       }
 
@@ -514,7 +595,7 @@ I arrived at that by trial and error, which is not always tenable, frankly.  Wha
 <pre class="source">COWRIE_HOME=~/cowrie
 for i in cowrie.1626302-1636522.json cowrie.16879981-16892488.json cowrie.21312194-21331475.json cowrie.698260-710913.json cowrie.762933-772239.json cowrie.929866-939552.json cowrie.1246880-1248235.json cowrie.19285959-19295444.json cowrie.16542668-16581213.json cowrie.5849832-5871517.json cowrie.6607473-6609163.json;do
   echo $i
-  cat $COWRIE_HOME/$i | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic cowrie
+  cat $COWRIE_HOME/$i | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic cowrie
   sleep 2
 done
 </pre></div></div>
@@ -582,7 +663,15 @@ done
 <li>177.238.236.21</li>
 <li>94.78.80.45</li>
 </ul>
-<p>Now we can look at <i>other</i> things that they&#x2019;re doing to build and refine our definition of what an alert is without resorting to hard-coding of rules.  Note that nothing in our enrichments actually used the string <tt>busybox</tt>, so this is a more general purpose way of navigating similar things.</p></div>
+<p>Now we can look at <i>other</i> things that they&#x2019;re doing to build and refine our definition of what an alert is without resorting to hard-coding of rules.  Note that nothing in our enrichments actually used the string <tt>busybox</tt>, so this is a more general purpose way of navigating similar things.</p>
+<div class="section">
+<h3><a name="Version_Info"></a>Version Info</h3>
+<p>Verified against:</p>
+<ul>
+
+<li>METRON_VERSION=0.5.0</li>
+<li>ELASTIC_VERSION=5.6.2</li>
+</ul></div></div>
         </div>
       </div>
     </div>

http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/use-cases/geographic_login_outliers/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/use-cases/geographic_login_outliers/index.html b/site/current-book/use-cases/geographic_login_outliers/index.html
index 2c0441a..34d25e1 100644
--- a/site/current-book/use-cases/geographic_login_outliers/index.html
+++ b/site/current-book/use-cases/geographic_login_outliers/index.html
@@ -1,13 +1,13 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/geographic_login_outliers/index.md at 2018-06-07
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/geographic_login_outliers/index.md at 2018-09-12
  | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Problem Statement</title>
     <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
       <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
       <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
     <li class="active ">Problem Statement</li>
-        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
-          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
         </ul>
       </div>
       <div class="row-fluid">
@@ -61,6 +61,7 @@
     <ul class="nav nav-list">
     <li><a href="../../use-cases/forensic_clustering/index.html" title="Forensic_clustering"><span class="none"></span>Forensic_clustering</a></li>
     <li class="active"><a href="#"><span class="none"></span>Geographic_login_outliers</a></li>
+    <li><a href="../../use-cases/parser_chaining/index.html" title="Parser_chaining"><span class="none"></span>Parser_chaining</a></li>
     <li><a href="../../use-cases/typosquat_detection/index.html" title="Typosquat_detection"><span class="none"></span>Typosquat_detection</a></li>
     </ul>
 </li>
@@ -233,7 +234,7 @@ if __name__ == '__main__':
     {
       &quot;profile&quot;: &quot;geo_distribution_from_centroid&quot;,
       &quot;foreach&quot;: &quot;'global'&quot;,
-      &quot;onlyif&quot;: &quot;exists(geo_distance) &amp;&amp; geo_distance != null&quot;,
+      &quot;onlyif&quot;: &quot;geo_distance != null&quot;,
       &quot;init&quot; : {
         &quot;s&quot;: &quot;STATS_INIT()&quot;
                },
@@ -245,7 +246,7 @@ if __name__ == '__main__':
     {
       &quot;profile&quot;: &quot;locations_by_user&quot;,
       &quot;foreach&quot;: &quot;user&quot;,
-      &quot;onlyif&quot;: &quot;exists(hash) &amp;&amp; hash != null &amp;&amp; LENGTH(hash) &gt; 0&quot;,
+      &quot;onlyif&quot;: &quot;hash != null &amp;&amp; LENGTH(hash) &gt; 0&quot;,
       &quot;init&quot; : {
         &quot;s&quot;: &quot;MULTISET_INIT()&quot;
                },
@@ -303,7 +304,6 @@ if __name__ == '__main__':
           &quot;dist_median := STATS_PERCENTILE(geo_distance_distr, 50.0)&quot;,
           &quot;dist_sd := STATS_SD(geo_distance_distr)&quot;,
           &quot;geo_outlier := ABS(dist_median - geo_distance) &gt;= 5*dist_sd&quot;,
-          &quot;is_alert := exists(is_alert) &amp;&amp; is_alert&quot;,
           &quot;is_alert := is_alert || (geo_outlier != null &amp;&amp; geo_outlier == true)&quot;,
           &quot;geo_distance_distr := null&quot;
         ]

http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/use-cases/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/use-cases/index.html b/site/current-book/use-cases/index.html
index 684cc14..d12b1ab 100644
--- a/site/current-book/use-cases/index.html
+++ b/site/current-book/use-cases/index.html
@@ -1,13 +1,13 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/index.md at 2018-06-07
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/use-cases/index.md at 2018-09-12
  | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta name="Date-Revision-yyyymmdd" content="20180912" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Worked Examples</title>
     <link rel="stylesheet" href="../css/apache-maven-fluido-1.7.min.css" />
@@ -32,8 +32,8 @@
       <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
       <li class=""><a href="../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
     <li class="active ">Worked Examples</li>
-        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
-          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-09-12</li>
+          <li id="projectVersion" class="pull-right">Version: 0.6.0</li>
         </ul>
       </div>
       <div class="row-fluid">
@@ -61,6 +61,7 @@
     <ul class="nav nav-list">
     <li><a href="../use-cases/forensic_clustering/index.html" title="Forensic_clustering"><span class="none"></span>Forensic_clustering</a></li>
     <li><a href="../use-cases/geographic_login_outliers/index.html" title="Geographic_login_outliers"><span class="none"></span>Geographic_login_outliers</a></li>
+    <li><a href="../use-cases/parser_chaining/index.html" title="Parser_chaining"><span class="none"></span>Parser_chaining</a></li>
     <li><a href="../use-cases/typosquat_detection/index.html" title="Typosquat_detection"><span class="none"></span>Typosquat_detection</a></li>
     </ul>
 </li>


Mime
View raw message