metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject [28/39] metron git commit: METRON-1607 update public web site to point at 0.5.0 new release (justinleet) closes apache/metron#1053
Date Fri, 08 Jun 2018 12:19:55 GMT
http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-analytics/metron-statistics/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-analytics/metron-statistics/index.html b/site/current-book/metron-analytics/metron-statistics/index.html
index 29084e8..0c3515d 100644
--- a/site/current-book/metron-analytics/metron-statistics/index.html
+++ b/site/current-book/metron-analytics/metron-statistics/index.html
@@ -1,756 +1,599 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia at 2018-01-03
- | Rendered using Apache Maven Fluido Skin 1.3.0
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-analytics/metron-statistics/index.md at 2018-06-07
+ | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180103" />
+    <meta name="Date-Revision-yyyymmdd" content="20180607" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Statistics and Mathematical Functions</title>
-    <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
+    <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
     <link rel="stylesheet" href="../../css/site.css" />
     <link rel="stylesheet" href="../../css/print.css" media="print" />
-
-      
-    <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script>
-
-                          
-        
-<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
-          
-            </head>
-        <body class="topBarDisabled">
-          
-                
-                    
-    
-        <div class="container-fluid">
-          <div id="banner">
-        <div class="pull-left">
-                                    <a href="http://metron.apache.org/" id="bannerLeft">
-                                                                                                <img src="../../images/metron-logo.png"  alt="Apache Metron" width="148px" height="48px"/>
-                </a>
-                      </div>
-        <div class="pull-right">  </div>
+    <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+              $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+            </script>
+  </head>
+  <body class="topBarDisabled">
+    <div class="container-fluid">
+      <div id="banner">
+        <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../images/metron-logo.png"  alt="Apache Metron" width="148px" height="48px"/></a></div>
+        <div class="pull-right"></div>
         <div class="clear"><hr/></div>
       </div>
 
       <div id="breadcrumbs">
         <ul class="breadcrumb">
-                
-                    
-                              <li class="">
-                    <a href="http://www.apache.org" class="externalLink" title="Apache">
-        Apache</a>
-        </li>
-      <li class="divider ">/</li>
-            <li class="">
-                    <a href="http://metron.apache.org/" class="externalLink" title="Metron">
-        Metron</a>
-        </li>
-      <li class="divider ">/</li>
-            <li class="">
-                    <a href="../../index.html" title="Documentation">
-        Documentation</a>
-        </li>
-      <li class="divider ">/</li>
-        <li class="">Statistics and Mathematical Functions</li>
-        
-                
-                    
-                  <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li>
-              <li id="projectVersion" class="pull-right">Version: 0.4.2</li>
-            
-                            </ul>
+      <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+      <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+      <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+    <li class="active ">Statistics and Mathematical Functions</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
+          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        </ul>
       </div>
-
-            
       <div class="row-fluid">
-        <div id="leftColumn" class="span3">
+        <div id="leftColumn" class="span2">
           <div class="well sidebar-nav">
-                
-                    
-                <ul class="nav nav-list">
-                    <li class="nav-header">User Documentation</li>
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     
                                                                          
-      <li>
-    
-                          <a href="../../index.html" title="Metron">
-          <i class="icon-chevron-down"></i>
-        Metron</a>
-                    <ul class="nav nav-list">
-                      
-      <li>
-    
-                          <a href="../../Upgrading.html" title="Upgrading">
-          <i class="none"></i>
-        Upgrading</a>
-            </li>
-                                                                                                                                                                
-      <li>
-    
-                          <a href="../../metron-analytics/index.html" title="Analytics">
-          <i class="icon-chevron-down"></i>
-        Analytics</a>
-                    <ul class="nav nav-list">
-                      
-      <li>
-    
-                          <a href="../../metron-analytics/metron-maas-service/index.html" title="Maas-service">
-          <i class="none"></i>
-        Maas-service</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-analytics/metron-profiler/index.html" title="Profiler">
-          <i class="none"></i>
-        Profiler</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-analytics/metron-profiler-client/index.html" title="Profiler-client">
-          <i class="none"></i>
-        Profiler-client</a>
-            </li>
-                                                                            
-      <li class="active">
-    
-            <a href="#"><i class="icon-chevron-down"></i>Statistics</a>
-                  <ul class="nav nav-list">
-                      
-      <li>
-    
-                          <a href="../../metron-analytics/metron-statistics/HLLP.html" title="HLLP">
-          <i class="none"></i>
-        HLLP</a>
-            </li>
-              </ul>
-        </li>
-              </ul>
-        </li>
-                      
-      <li>
-    
-                          <a href="../../metron-contrib/metron-docker/index.html" title="Docker">
-          <i class="none"></i>
-        Docker</a>
-            </li>
-                                                                                                                                                                                                                                                                                                                                                                                                            
-      <li>
-    
-                          <a href="../../metron-deployment/index.html" title="Deployment">
-          <i class="icon-chevron-right"></i>
-        Deployment</a>
-                  </li>
-                      
-      <li>
-    
-                          <a href="../../metron-interface/metron-alerts/index.html" title="Alerts">
-          <i class="none"></i>
-        Alerts</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-interface/metron-config/index.html" title="Config">
-          <i class="none"></i>
-        Config</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-interface/metron-rest/index.html" title="Rest">
-          <i class="none"></i>
-        Rest</a>
-            </li>
-                                                                                                                                                                                                                                                                                    
-      <li>
-    
-                          <a href="../../metron-platform/index.html" title="Platform">
-          <i class="icon-chevron-right"></i>
-        Platform</a>
-                  </li>
-                                                                                          
-      <li>
-    
-                          <a href="../../metron-sensors/index.html" title="Sensors">
-          <i class="icon-chevron-right"></i>
-        Sensors</a>
-                  </li>
-                      
-      <li>
-    
-                          <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example">
-          <i class="none"></i>
-        Stellar-3rd-party-example</a>
-            </li>
-                                                                        
-      <li>
-    
-                          <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common">
-          <i class="icon-chevron-right"></i>
-        Stellar-common</a>
-                  </li>
-                                                                                          
-      <li>
-    
-                          <a href="../../use-cases/index.html" title="Use-cases">
-          <i class="icon-chevron-right"></i>
-        Use-cases</a>
-                  </li>
-              </ul>
-        </li>
-            </ul>
-                
-                    
-                
-          <hr class="divider" />
-
-           <div id="poweredBy">
-                            <div class="clear"></div>
-                            <div class="clear"></div>
-                            <div class="clear"></div>
-                             <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
-        <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" />
-      </a>
-                  </div>
+    <ul class="nav nav-list">
+      <li class="nav-header">User Documentation</li>
+    <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+    <ul class="nav nav-list">
+    <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+    <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+    <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-down"></span>Analytics</a>
+    <ul class="nav nav-list">
+    <li><a href="../../metron-analytics/metron-maas-service/index.html" title="Maas-service"><span class="none"></span>Maas-service</a></li>
+    <li><a href="../../metron-analytics/metron-profiler/index.html" title="Profiler"><span class="none"></span>Profiler</a></li>
+    <li><a href="../../metron-analytics/metron-profiler-client/index.html" title="Profiler-client"><span class="none"></span>Profiler-client</a></li>
+    <li class="active"><a href="#"><span class="icon-chevron-down"></span>Statistics</a>
+    <ul class="nav nav-list">
+    <li><a href="../../metron-analytics/metron-statistics/HLLP.html" title="HLLP"><span class="none"></span>HLLP</a></li>
+    </ul>
+</li>
+    </ul>
+</li>
+    <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+    <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+    <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+    <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
+    <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
+    <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+    <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
+    <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+    <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+    <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+    <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+    <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+    </ul>
+</li>
+</ul>
+          <hr />
+          <div id="poweredBy">
+            <div class="clear"></div>
+            <div class="clear"></div>
+            <div class="clear"></div>
+            <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a>
+            </div>
           </div>
         </div>
-        
-                
-        <div id="bodyColumn"  class="span9" >
-                                  
-            <h1>Statistics and Mathematical Functions</h1>
+        <div id="bodyColumn"  class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Statistics and Mathematical Functions</h1>
 <p><a name="Statistics_and_Mathematical_Functions"></a></p>
-<p>A variety of non-trivial and advanced analytics make use of statistics and advanced mathematical functions. Particular, capturing the statistical snapshots in a scalable way can open up doors for more advanced analytics such as outlier analysis. As such, this project is aimed at capturing a robust set of statistical functions and statistical-based algorithms in the form of Stellar functions. These functions can be used from everywhere where Stellar is used.</p>
+<p>A variety of non-trivial and advanced analytics make use of statistics and advanced mathematical functions.  Particular, capturing the statistical snapshots in a scalable way can open up doors for more advanced analytics such as outlier analysis.  As such, this project is aimed at capturing a robust set of statistical functions and statistical-based algorithms in the form of Stellar functions.  These functions can be used from everywhere where Stellar is used.</p>
 <div class="section">
 <h2><a name="Stellar_Functions"></a>Stellar Functions</h2>
 <div class="section">
 <h3><a name="Approximation_Statistics"></a>Approximation Statistics</h3>
 <div class="section">
 <h4><a name="HLLP_ADD"></a><tt>HLLP_ADD</tt></h4>
-
 <ul>
-  
+
 <li>Description: Add value to the HyperLogLogPlus estimator set. See <a href="HLLP.html">HLLP README</a></li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>hyperLogLogPlus - the hllp estimator to add a value to</li>
-    
 <li>value+ - value to add to the set. Takes a single item or a list.</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The HyperLogLogPlus set with a new value added</li>
 </ul></div>
 <div class="section">
 <h4><a name="HLLP_CARDINALITY"></a><tt>HLLP_CARDINALITY</tt></h4>
-
 <ul>
-  
+
 <li>Description: Returns HyperLogLogPlus-estimated cardinality for this set. See <a href="HLLP.html">HLLP README</a></li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>hyperLogLogPlus - the hllp set</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: Long value representing the cardinality for this set. Cardinality of a null set is 0.</li>
 </ul></div>
 <div class="section">
 <h4><a name="HLLP_INIT"></a><tt>HLLP_INIT</tt></h4>
-
 <ul>
-  
+
 <li>Description: Initializes the HyperLogLogPlus estimator set. p must be a value between 4 and sp and sp must be less than 32 and greater than 4. See <a href="HLLP.html">HLLP README</a></li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>p - the precision value for the normal set</li>
-    
 <li>sp - the precision value for the sparse set. If p is set, but sp is 0 or not specified, the sparse set will be disabled.</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: A new HyperLogLogPlus set</li>
 </ul></div>
 <div class="section">
 <h4><a name="HLLP_MERGE"></a><tt>HLLP_MERGE</tt></h4>
-
 <ul>
-  
+
 <li>Description: Merge hllp sets together. The resulting estimator is initialized with p and sp precision values from the first provided hllp estimator set. See <a href="HLLP.html">HLLP README</a></li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>hllp - List of hllp estimators to merge. Takes a single hllp set or a list.</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: A new merged HyperLogLogPlus estimator set. Passing an empty list returns null.</li>
 </ul></div></div>
 <div class="section">
 <h3><a name="Mathematical_Functions"></a>Mathematical Functions</h3>
 <div class="section">
 <h4><a name="ABS"></a><tt>ABS</tt></h4>
-
 <ul>
-  
+
 <li>Description: Returns the absolute value of a number.</li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>number - The number to take the absolute value of</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The absolute value of the number passed in.</li>
 </ul></div>
 <div class="section">
 <h4><a name="BIN"></a><tt>BIN</tt></h4>
-
 <ul>
-  
+
 <li>Description: Computes the bin that the value is in given a set of bounds.</li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>value - The value to bin</li>
-    
 <li>bounds - A list of value bounds (excluding min and max) in sorted order.</li>
-  </ul></li>
-  
-<li>Returns: Which bin N the value falls in such that bound(N-1) &lt; value &lt;= bound(N). No min and max bounds are provided, so values smaller than the 0&#x2019;th bound go in the 0&#x2019;th bin, and values greater than the last bound go in the M&#x2019;th bin.</li>
+</ul>
+</li>
+<li>Returns: Which bin N the value falls in such that bound(N-1) &lt; value &lt;= bound(N).  No min and max bounds are provided, so values smaller than the 0&#x2019;th bound go in the 0&#x2019;th bin, and values greater than the last bound go in the M&#x2019;th bin.</li>
+</ul></div></div>
+<div class="section">
+<h3><a name="Sampling_Functions"></a>Sampling Functions</h3>
+<div class="section">
+<h4><a name="SAMPLE_ADD"></a><tt>SAMPLE_ADD</tt></h4>
+<ul>
+
+<li>Description: Add a value or collection of values to a sampler.</li>
+<li>Input:
+<ul>
+
+<li>sampler - Sampler to use.  If null, then a default Uniform sampler is created.</li>
+<li>o - The value to add.  If o is an Iterable, then each item is added.</li>
+</ul>
+</li>
+<li>Returns: The sampler.</li>
+</ul></div>
+<div class="section">
+<h4><a name="SAMPLE_GET"></a><tt>SAMPLE_GET</tt></h4>
+<ul>
+
+<li>Description: Return the sample.</li>
+<li>Input:
+<ul>
+
+<li>sampler - Sampler to use.</li>
+</ul>
+</li>
+<li>Returns: The resulting sample.</li>
+</ul></div>
+<div class="section">
+<h4><a name="SAMPLE_INIT"></a><tt>SAMPLE_INIT</tt></h4>
+<ul>
+
+<li>Description: Create a <a class="externalLink" href="https://en.wikipedia.org/wiki/Reservoir_sampling">reservoir sampler</a> of a specific size or, if unspecified, size 1024.  Elements sampled by the reservoir sampler will be included in the final sample with equal probability.</li>
+<li>Input:
+<ul>
+
+<li>size? - The size of the reservoir sampler.  If unspecified, the size is 1024.</li>
+</ul>
+</li>
+<li>Returns: The sampler object.</li>
+</ul></div>
+<div class="section">
+<h4><a name="SAMPLE_MERGE"></a><tt>SAMPLE_MERGE</tt></h4>
+<ul>
+
+<li>Description: Merge and resample a collection of samples.</li>
+<li>Input:
+<ul>
+
+<li>samplers - A list of samplers to merge.</li>
+</ul>
+</li>
+<li>Returns: A sampler which represents the resampled merger of the samplers.</li>
 </ul></div></div>
 <div class="section">
 <h3><a name="Distributional_Statistics"></a>Distributional Statistics</h3>
 <div class="section">
 <h4><a name="STATS_ADD"></a><tt>STATS_ADD</tt></h4>
-
 <ul>
-  
+
 <li>Description: Adds one or more input values to those that are used to calculate the summary statistics.</li>
-  
 <li>Input:
-  
 <ul>
-    
-<li>stats - The Stellar statistics object. If null, then a new one is initialized.</li>
-    
+
+<li>stats - The Stellar statistics object.  If null, then a new one is initialized.</li>
 <li>value+ - One or more items to add. Each item may be a number or a list of numbers. If an item is a list, each number in the list will be added.</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: A Stellar statistics object</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_BIN"></a><tt>STATS_BIN</tt></h4>
-
 <ul>
-  
+
 <li>Description: Computes the bin that the value is in based on the statistical distribution.</li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-    
 <li>value - The value to bin</li>
-    
-<li>bounds? - A list of percentile bin bounds (excluding min and max) or a string representing a known and common set of bins. For convenience, we have provided QUARTILE, QUINTILE, and DECILE which you can pass in as a string arg. If this argument is omitted, then we assume a Quartile bin split.</li>
-  </ul></li>
-  
+<li>bounds? - A list of percentile bin bounds (excluding min and max) or a string representing a known and common set of bins.  For convenience, we have provided QUARTILE, QUINTILE, and DECILE which you can pass in as a string arg. If this argument is omitted, then we assume a Quartile bin split.</li>
+</ul>
+</li>
 <li>Returns: &quot;Which bin N the value falls in such that bound(N-1) &lt; value &lt;= bound(N). No min and max bounds are provided, so values smaller than the 0&#x2019;th bound go in the 0&#x2019;th bin, and values greater than the last bound go in the M&#x2019;th bin.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_COUNT"></a><tt>STATS_COUNT</tt></h4>
-
 <ul>
-  
+
 <li>Description: Calculates the count of the values accumulated (or in the window if a window is used).</li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The count of the values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_GEOMETRIC_MEAN"></a><tt>STATS_GEOMETRIC_MEAN</tt></h4>
-
 <ul>
-  
+
 <li>Description: Calculates the geometric mean of the accumulated values (or in the window if a window is used). See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The geometric mean of the values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_INIT"></a><tt>STATS_INIT</tt></h4>
-
 <ul>
-  
+
 <li>Description: Initializes a statistics object</li>
-  
 <li>Input:
-  
 <ul>
-    
-<li>window_size - The number of input data values to maintain in a rolling window in memory. If window_size is equal to 0, then no rolling window is maintained. Using no rolling window is less memory intensive, but cannot calculate certain statistics like percentiles and kurtosis.</li>
-  </ul></li>
-  
+
+<li>window_size - The number of input data values to maintain in a rolling window in memory.  If window_size is equal to 0, then no rolling window is maintained. Using no rolling window is less memory intensive, but cannot calculate certain statistics like percentiles and kurtosis.</li>
+</ul>
+</li>
 <li>Returns: A Stellar statistics object</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_KURTOSIS"></a><tt>STATS_KURTOSIS</tt></h4>
-
 <ul>
-  
-<li>Description: Calculates the kurtosis of the accumulated values (or in the window if a window is used). See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
-  
+
+<li>Description: Calculates the kurtosis of the accumulated values (or in the window if a window is used).  See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The kurtosis of the values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_MAX"></a><tt>STATS_MAX</tt></h4>
-
 <ul>
-  
+
 <li>Description: Calculates the maximum of the accumulated values (or in the window if a window is used).</li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The maximum of the accumulated values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_MEAN"></a><tt>STATS_MEAN</tt></h4>
-
 <ul>
-  
+
 <li>Description: Calculates the mean of the accumulated values (or in the window if a window is used).</li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The mean of the values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_MERGE"></a><tt>STATS_MERGE</tt></h4>
-
 <ul>
-  
+
 <li>Description: Merges statistics objects.</li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>statistics - A list of statistics objects</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: A Stellar statistics object</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_MIN"></a><tt>STATS_MIN</tt></h4>
-
 <ul>
-  
+
 <li>Description: Calculates the minimum of the accumulated values (or in the window if a window is used).</li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The minimum of the accumulated values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_PERCENTILE"></a><tt>STATS_PERCENTILE</tt></h4>
-
 <ul>
-  
+
 <li>Description: Computes the p&#x2019;th percentile of the accumulated values (or in the window if a window is used).</li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-    
 <li>p - a double where 0 &lt;= p &lt; 1 representing the percentile</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The p&#x2019;th percentile of the data or NaN if the statistics object is null</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_POPULATION_VARIANCE"></a><tt>STATS_POPULATION_VARIANCE</tt></h4>
-
 <ul>
-  
-<li>Description: Calculates the population variance of the accumulated values (or in the window if a window is used). See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
-  
+
+<li>Description: Calculates the population variance of the accumulated values (or in the window if a window is used).  See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The population variance of the values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_QUADRATIC_MEAN"></a><tt>STATS_QUADRATIC_MEAN</tt></h4>
-
 <ul>
-  
-<li>Description: Calculates the quadratic mean of the accumulated values (or in the window if a window is used). See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
-  
+
+<li>Description: Calculates the quadratic mean of the accumulated values (or in the window if a window is used).  See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The quadratic mean of the values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_SD"></a><tt>STATS_SD</tt></h4>
-
 <ul>
-  
-<li>Description: Calculates the standard deviation of the accumulated values (or in the window if a window is used). See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
-  
+
+<li>Description: Calculates the standard deviation of the accumulated values (or in the window if a window is used).  See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The standard deviation of the values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_SKEWNESS"></a><tt>STATS_SKEWNESS</tt></h4>
-
 <ul>
-  
-<li>Description: Calculates the skewness of the accumulated values (or in the window if a window is used). See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
-  
+
+<li>Description: Calculates the skewness of the accumulated values (or in the window if a window is used).  See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The skewness of the values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_SUM"></a><tt>STATS_SUM</tt></h4>
-
 <ul>
-  
+
 <li>Description: Calculates the sum of the accumulated values (or in the window if a window is used).</li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The sum of the values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_SUM_LOGS"></a><tt>STATS_SUM_LOGS</tt></h4>
-
 <ul>
-  
-<li>Description: Calculates the sum of the (natural) log of the accumulated values (or in the window if a window is used). See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
-  
+
+<li>Description: Calculates the sum of the (natural) log of the accumulated values (or in the window if a window is used).  See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The sum of the (natural) log of the values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_SUM_SQUARES"></a><tt>STATS_SUM_SQUARES</tt></h4>
-
 <ul>
-  
+
 <li>Description: Calculates the sum of the squares of the accumulated values (or in the window if a window is used).</li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The sum of the squares of the values in the window or NaN if the statistics object is null.</li>
 </ul></div>
 <div class="section">
 <h4><a name="STATS_VARIANCE"></a><tt>STATS_VARIANCE</tt></h4>
-
 <ul>
-  
-<li>Description: Calculates the variance of the accumulated values (or in the window if a window is used). See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
-  
+
+<li>Description: Calculates the variance of the accumulated values (or in the window if a window is used).  See <a class="externalLink" href="http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics">http://commons.apache.org/proper/commons-math/userguide/stat.html#a1.2_Descriptive_statistics</a></li>
 <li>Input:
-  
 <ul>
-    
+
 <li>stats - The Stellar statistics object</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The variance of the values in the window or NaN if the statistics object is null.</li>
 </ul></div></div>
 <div class="section">
+<h3><a name="Information_Theory_Functions"></a>Information Theory Functions</h3>
+<div class="section">
+<h4><a name="IT_ENTROPY"></a><tt>IT_ENTROPY</tt></h4>
+<ul>
+
+<li>Description: Computes the base-2 entropy of a multiset</li>
+<li>Input:
+<ul>
+
+<li>input - a multiset (a map of objects to counts).</li>
+</ul>
+</li>
+<li>Returns: The <a class="externalLink" href="https://en.wikipedia.org/wiki/Entropy_(information_theory)#Definition">base-2 entropy</a> of the count .  The unit of this is bits.</li>
+</ul></div></div>
+<div class="section">
 <h3><a name="Statistical_Outlier_Detection"></a>Statistical Outlier Detection</h3>
 <div class="section">
 <h4><a name="OUTLIER_MAD_STATE_MERGE"></a><tt>OUTLIER_MAD_STATE_MERGE</tt></h4>
-
 <ul>
-  
+
 <li>Description: Update the statistical state required to compute the Median Absolute Deviation.</li>
-  
 <li>Input:
-  
 <ul>
-    
-<li>[state] - A list of Median Absolute Deviation States to merge. Generally these are states across time.</li>
-    
+
+<li>[state] - A list of Median Absolute Deviation States to merge.  Generally these are states across time.</li>
 <li>currentState? - The current state (optional)</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The Median Absolute Deviation state</li>
 </ul></div>
 <div class="section">
 <h4><a name="OUTLIER_MAD_ADD"></a><tt>OUTLIER_MAD_ADD</tt></h4>
-
 <ul>
-  
+
 <li>Description: Add a piece of data to the state.</li>
-  
 <li>Input:
-  
 <ul>
-    
+
 <li>state - The MAD state</li>
-    
 <li>value - The numeric value to add</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Returns: The MAD state</li>
 </ul></div>
 <div class="section">
 <h4><a name="OUTLIER_MAD_SCORE"></a><tt>OUTLIER_MAD_SCORE</tt></h4>
-
 <ul>
-  
-<li>Description: Get the modified z-score normalized by the MAD: scale * | x_i - median(X) | / MAD. See the first page of <a class="externalLink" href="http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/BetterThanMAD.pdf">http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/BetterThanMAD.pdf</a></li>
-  
+
+<li>Description: Get the modified z-score normalized by the MAD: scale * | x_i - median(X) | / MAD.  See the first page of <a class="externalLink" href="http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/BetterThanMAD.pdf">http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/BetterThanMAD.pdf</a></li>
 <li>Input:
-  
 <ul>
-    
+
 <li>state - The MAD state</li>
-    
 <li>value - The numeric value to score</li>
-    
-<li>scale? - Optionally the scale to use when computing the modified z-score. Default is <tt>0.6745</tt>, see the first page of <a class="externalLink" href="http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/BetterThanMAD.pdf">http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/BetterThanMAD.pdf</a></li>
-  </ul></li>
-  
+<li>scale? - Optionally the scale to use when computing the modified z-score.  Default is <tt>0.6745</tt>, see the first page of <a class="externalLink" href="http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/BetterThanMAD.pdf">http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/BetterThanMAD.pdf</a></li>
+</ul>
+</li>
 <li>Returns: The modified z-score</li>
 </ul>
 <p><a name="Outlier_Analysis"></a></p>
 <h1>Outlier Analysis</h1>
-<p>A common desire is to find anomalies in numerical data. To that end, we have some simple statistical anomaly detectors.</p></div></div></div>
+<p>A common desire is to find anomalies in numerical data.  To that end, we have some simple statistical anomaly detectors.</p></div></div></div>
 <div class="section">
 <h2><a name="Median_Absolute_Deviation"></a>Median Absolute Deviation</h2>
-<p>Much has been written about this robust estimator. See the first page of <a class="externalLink" href="http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/BetterThanMAD.pdf">http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/BetterThanMAD.pdf</a> for a good coverage of the good and the bad of MAD. The usage, however is fairly straightforward:</p>
-
+<p>Much has been written about this robust estimator.  See the first page of <a class="externalLink" href="http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/BetterThanMAD.pdf">http://web.ipac.caltech.edu/staff/fmasci/home/astro_refs/BetterThanMAD.pdf</a> for a good coverage of the good and the bad of MAD.  The usage, however is fairly straightforward:</p>
 <ul>
-  
+
 <li>Gather the statistical state required to compute the MAD
-  
 <ul>
-    
+
 <li>The distribution of the values of a univariate random variable over time.</li>
-    
 <li>The distribution of the absolute deviations of the values from the median.</li>
-  </ul></li>
-  
-<li>Use this statistical state to score unseen values. The higher the score, the more unlike the previously seen data the value is.</li>
 </ul>
-<p>There are a couple of issues which make MAD a bit hard to compute. First, the statistical state requires computing median, which can be computationally expensive to compute exactly. To get around this, we use the OnlineStatisticalProvider to compute a sketch rather than the exact median. Secondly, the statistical state for seasonal data should be limited to a fixed, trailing window. We do this by ensuring that the MAD state is mergeable and able to be queried from within the Profiler.</p>
+</li>
+<li>Use this statistical state to score unseen values.  The higher the score, the more unlike the previously seen data the value is.</li>
+</ul>
+<p>There are a couple of issues which make MAD a bit hard to compute. First, the statistical state requires computing median, which can be computationally expensive to compute exactly.  To get around this, we use the OnlineStatisticalProvider to compute a sketch rather than the exact median.  Secondly, the statistical state for seasonal data should be limited to a fixed, trailing window.  We do this by ensuring that the MAD state is mergeable and able to be queried from within the Profiler.</p>
 <div class="section">
 <h3><a name="Example"></a>Example</h3>
 <p>We will create a dummy data stream of gaussian noise to illustrate how to use the MAD functionality along with the profiler to tag messages as outliers or not.</p>
-<p>To do this, we will create a </p>
-
+<p>To do this, we will create a</p>
 <ul>
-  
+
 <li>data generator</li>
-  
 <li>parser</li>
-  
 <li>profiler profile</li>
-  
 <li>enrichment and threat triage</li>
 </ul>
 <div class="section">
 <h4><a name="Data_Generator"></a>Data Generator</h4>
 <p>We can create a simple python script to generate a stream of gaussian noise at the frequency of one message per second as a python script which should be saved at <tt>~/rand_gen.py</tt>:</p>
 
-<div class="source">
-<div class="source">
-<pre>#!/usr/bin/python
+<div>
+<div>
+<pre class="source">#!/usr/bin/python
 import random
 import sys
 import time
@@ -766,21 +609,19 @@ def main():
 if __name__ == '__main__':
   main()
 </pre></div></div>
-<p>This script will take the following as arguments:</p>
 
+<p>This script will take the following as arguments:</p>
 <ul>
-  
+
 <li>The mean of the data generated</li>
-  
 <li>The standard deviation of the data generated</li>
-  
 <li>The frequency (in seconds) of the data generated</li>
 </ul>
 <p>If, however, you&#x2019;d like to test a longer tailed distribution, like the student t-distribution and have numpy installed, you can use the following as <tt>~/rand_gen.py</tt>:</p>
 
-<div class="source">
-<div class="source">
-<pre>#!/usr/bin/python
+<div>
+<div>
+<pre class="source">#!/usr/bin/python
 import random
 import sys
 import time
@@ -797,12 +638,11 @@ def main():
 if __name__ == '__main__':
   main()
 </pre></div></div>
-<p>This script will take the following as arguments:</p>
 
+<p>This script will take the following as arguments:</p>
 <ul>
-  
+
 <li>The degrees of freedom for the distribution</li>
-  
 <li>The frequency (in seconds) of the data generated</li>
 </ul></div>
 <div class="section">
@@ -810,9 +650,9 @@ if __name__ == '__main__':
 <p>We will create a parser that will take the single numbers in and create a message with a field called <tt>value</tt> in them using the <tt>CSVParser</tt>.</p>
 <p>Add the following file to <tt>$METRON_HOME/config/zookeeper/parsers/mad.json</tt>:</p>
 
-<div class="source">
-<div class="source">
-<pre>{
+<div>
+<div>
+<pre class="source">{
   &quot;parserClassName&quot; : &quot;org.apache.metron.parsers.csv.CSVParser&quot;
  ,&quot;sensorTopic&quot; : &quot;mad&quot;
  ,&quot;parserConfig&quot; : {
@@ -830,15 +670,16 @@ if __name__ == '__main__':
     }
                            ]
 }
-</pre></div></div></div>
+</pre></div></div>
+</div>
 <div class="section">
 <h4><a name="Enrichment_and_Threat_Intel"></a>Enrichment and Threat Intel</h4>
-<p>We will set a threat triage level of <tt>10</tt> if a message generates a outlier score of more than 3.5. This cutoff will depend on your data and should be adjusted based on the assumed underlying distribution. Note that under the assumptions of normality, MAD will act as a robust estimator of the standard deviation, so the cutoff should be considered the number of standard deviations away. For other distributions, there are other interpretations which will make sense in the context of measuring the &#x201c;degree different&#x201d;. See <a class="externalLink" href="http://eurekastatistics.com/using-the-median-absolute-deviation-to-find-outliers/">http://eurekastatistics.com/using-the-median-absolute-deviation-to-find-outliers/</a> for a brief discussion of this.</p>
+<p>We will set a threat triage level of <tt>10</tt> if a message generates a outlier score of more than 3.5. This cutoff will depend on your data and should be adjusted based on the assumed underlying distribution.  Note that under the assumptions of normality, MAD will act as a robust estimator of the standard deviation, so the cutoff should be considered the number of standard deviations away.  For other distributions, there are other interpretations which will make sense in the context of measuring the &#x201c;degree different&#x201d;.  See <a class="externalLink" href="http://eurekastatistics.com/using-the-median-absolute-deviation-to-find-outliers/">http://eurekastatistics.com/using-the-median-absolute-deviation-to-find-outliers/</a> for a brief discussion of this.</p>
 <p>Create the following in <tt>$METRON_HOME/config/zookeeper/enrichments/mad.json</tt>:</p>
 
-<div class="source">
-<div class="source">
-<pre>{
+<div>
+<div>
+<pre class="source">{
   &quot;enrichment&quot;: {
     &quot;fieldMap&quot;: {
       &quot;stellar&quot; : {
@@ -866,11 +707,12 @@ PROFILE_GET( 'sketchy_mad', 'global', PROFILE_FIXED(10, 'MINUTES')) ), value)&qu
   }
 }
 </pre></div></div>
+
 <p>We also need an indexing configuration. Create the following in <tt>$METRON_HOME/config/zookeeper/indexing/mad.json</tt>:</p>
 
-<div class="source">
-<div class="source">
-<pre>{
+<div>
+<div>
+<pre class="source">{
   &quot;hdfs&quot; : {
     &quot;index&quot;: &quot;mad&quot;,
     &quot;batchSize&quot;: 1,
@@ -882,15 +724,16 @@ PROFILE_GET( 'sketchy_mad', 'global', PROFILE_FIXED(10, 'MINUTES')) ), value)&qu
     &quot;enabled&quot; : true
   }
 }
-</pre></div></div></div>
+</pre></div></div>
+</div>
 <div class="section">
 <h4><a name="The_Profiler"></a>The Profiler</h4>
-<p>We can set up the profiler to track the MAD statistical state required to compute MAD. For the purposes of this demonstration, we will configure the profiler to capture statistics on the minute mark. We will capture a global statistical state for the <tt>value</tt> field and we will look back for a 5 minute window when computing the median.</p>
+<p>We can set up the profiler to track the MAD statistical state required to compute MAD.  For the purposes of this demonstration, we will configure the profiler to capture statistics on the minute mark.  We will capture a global statistical state for the <tt>value</tt> field and we will look back for a 5 minute window when computing the median.</p>
 <p>Create the following file at <tt>$METRON_HOME/config/zookeeper/profiler.json</tt>:</p>
 
-<div class="source">
-<div class="source">
-<pre>{
+<div>
+<div>
+<pre class="source">{
   &quot;profiles&quot;: [
     {
       &quot;profile&quot;: &quot;sketchy_mad&quot;,
@@ -908,64 +751,69 @@ PROFILE_GET( 'sketchy_mad', 'global', PROFILE_FIXED(10, 'MINUTES')) ), value)&qu
   ]
 }
 </pre></div></div>
+
 <p>Adjust <tt>$METRON_HOME/config/zookeeper/global.json</tt> to adjust the capture duration:</p>
 
-<div class="source">
-<div class="source">
-<pre> &quot;profiler.client.period.duration&quot; : &quot;1&quot;,
+<div>
+<div>
+<pre class="source"> &quot;profiler.client.period.duration&quot; : &quot;1&quot;,
  &quot;profiler.client.period.duration.units&quot; : &quot;MINUTES&quot;
 </pre></div></div>
+
 <p>Adjust <tt>$METRON_HOME/config/profiler.properties</tt> to adjust the capture duration by changing <tt>profiler.period.duration=15</tt> to <tt>profiler.period.duration=1</tt></p></div>
 <div class="section">
 <h4><a name="Execute_the_Flow"></a>Execute the Flow</h4>
-
 <ol style="list-style-type: decimal">
-  
+
 <li>
-<p>Install the elasticsearch head plugin by executing: <tt>/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head</tt></p></li>
-  
+
+<p>Install the elasticsearch head plugin by executing: <tt>/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head</tt></p>
+</li>
 <li>
-<p>Stopping all other parser topologies via monit</p></li>
-  
+
+<p>Stopping all other parser topologies via monit</p>
+</li>
 <li>
-<p>Create the <tt>mad</tt> kafka topic by executing: <tt>/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper node1:2181 --create --topic mad --partitions 1 --replication-factor 1</tt></p></li>
-  
+
+<p>Create the <tt>mad</tt> kafka topic by executing: <tt>/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper node1:2181 --create --topic mad --partitions 1 --replication-factor 1</tt></p>
+</li>
 <li>
-<p>Push the modified configs by executing: <tt>$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -z node1:2181 -i $METRON_HOME/config/zookeeper/</tt></p></li>
-  
+
+<p>Push the modified configs by executing: <tt>$METRON_HOME/bin/zk_load_configs.sh --mode PUSH -z node1:2181 -i $METRON_HOME/config/zookeeper/</tt></p>
+</li>
 <li>
-<p>Start the profiler by executing: <tt>$METRON_HOME/bin/start_profiler_topology.sh</tt></p></li>
-  
+
+<p>Start the profiler by executing: <tt>$METRON_HOME/bin/start_profiler_topology.sh</tt></p>
+</li>
 <li>
-<p>Start the parser topology by executing: <tt>$METRON_HOME/bin/start_parser_topology.sh -k node1:6667 -z node1:2181 -s mad</tt></p></li>
-  
+
+<p>Start the parser topology by executing: <tt>$METRON_HOME/bin/start_parser_topology.sh -k node1:6667 -z node1:2181 -s mad</tt></p>
+</li>
 <li>
-<p>Ensure that the enrichment and indexing topologies are started. If not, then start those via monit or by hand.</p></li>
-  
+
+<p>Ensure that the enrichment and indexing topologies are started.  If not, then start those via monit or by hand.</p>
+</li>
 <li>
-<p>Generate data into kafka by executing the following for at least 10 minutes: <tt>~/rand_gen.py 0 1 1 | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic mad</tt> Note: if you chose the use the t-distribution script above, you would adjust the parameters of the <tt>rand_gen.py</tt> script accordingly.</p></li>
-  
+
+<p>Generate data into kafka by executing the following for at least 10 minutes: <tt>~/rand_gen.py 0 1 1 | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic mad</tt> Note: if you chose the use the t-distribution script above, you would adjust the parameters of the <tt>rand_gen.py</tt> script accordingly.</p>
+</li>
 <li>
-<p>Stop the above with ctrl-c and send in an obvious outlier into kafka: <tt>echo &quot;1000&quot; | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic mad</tt></p></li>
+
+<p>Stop the above with ctrl-c and send in an obvious outlier into kafka: <tt>echo &quot;1000&quot; | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list node1:6667 --topic mad</tt></p>
+</li>
 </ol>
 <p>You should be able to find the outlier via the elasticsearch head plugin by searching for the messages where <tt>is_alert</tt> is <tt>true</tt>.</p></div></div></div>
-                  </div>
-            </div>
-          </div>
-
+        </div>
+      </div>
+    </div>
     <hr/>
-
     <footer>
-            <div class="container-fluid">
-              <div class="row span12">Copyright &copy;                    2018
-                        <a href="https://www.apache.org">The Apache Software Foundation</a>.
-            All Rights Reserved.      
-                    
+      <div class="container-fluid">
+        <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+            and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+        </div>
       </div>
-
-                          
-        
-                </div>
     </footer>
   </body>
 </html>

http://git-wip-us.apache.org/repos/asf/metron/blob/ae1d3eb9/site/current-book/metron-contrib/metron-docker/index.html
----------------------------------------------------------------------
diff --git a/site/current-book/metron-contrib/metron-docker/index.html b/site/current-book/metron-contrib/metron-docker/index.html
index 761c3b0..5769043 100644
--- a/site/current-book/metron-contrib/metron-docker/index.html
+++ b/site/current-book/metron-contrib/metron-docker/index.html
@@ -1,274 +1,170 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia at 2018-01-03
- | Rendered using Apache Maven Fluido Skin 1.3.0
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-contrib/metron-docker/index.md at 2018-06-07
+ | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180103" />
+    <meta name="Date-Revision-yyyymmdd" content="20180607" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Metron Docker</title>
-    <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
+    <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
     <link rel="stylesheet" href="../../css/site.css" />
     <link rel="stylesheet" href="../../css/print.css" media="print" />
-
-      
-    <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script>
-
-                          
-        
-<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
-          
-            </head>
-        <body class="topBarDisabled">
-          
-                
-                    
-    
-        <div class="container-fluid">
-          <div id="banner">
-        <div class="pull-left">
-                                    <a href="http://metron.apache.org/" id="bannerLeft">
-                                                                                                <img src="../../images/metron-logo.png"  alt="Apache Metron" width="148px" height="48px"/>
-                </a>
-                      </div>
-        <div class="pull-right">  </div>
+    <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+              $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+            </script>
+  </head>
+  <body class="topBarDisabled">
+    <div class="container-fluid">
+      <div id="banner">
+        <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../images/metron-logo.png"  alt="Apache Metron" width="148px" height="48px"/></a></div>
+        <div class="pull-right"></div>
         <div class="clear"><hr/></div>
       </div>
 
       <div id="breadcrumbs">
         <ul class="breadcrumb">
-                
-                    
-                              <li class="">
-                    <a href="http://www.apache.org" class="externalLink" title="Apache">
-        Apache</a>
-        </li>
-      <li class="divider ">/</li>
-            <li class="">
-                    <a href="http://metron.apache.org/" class="externalLink" title="Metron">
-        Metron</a>
-        </li>
-      <li class="divider ">/</li>
-            <li class="">
-                    <a href="../../index.html" title="Documentation">
-        Documentation</a>
-        </li>
-      <li class="divider ">/</li>
-        <li class="">Metron Docker</li>
-        
-                
-                    
-                  <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li>
-              <li id="projectVersion" class="pull-right">Version: 0.4.2</li>
-            
-                            </ul>
+      <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+      <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+      <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+    <li class="active ">Metron Docker</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
+          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        </ul>
       </div>
-
-            
       <div class="row-fluid">
-        <div id="leftColumn" class="span3">
+        <div id="leftColumn" class="span2">
           <div class="well sidebar-nav">
-                
-                    
-                <ul class="nav nav-list">
-                    <li class="nav-header">User Documentation</li>
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     
                                                                          
-      <li>
-    
-                          <a href="../../index.html" title="Metron">
-          <i class="icon-chevron-down"></i>
-        Metron</a>
-                    <ul class="nav nav-list">
-                      
-      <li>
-    
-                          <a href="../../Upgrading.html" title="Upgrading">
-          <i class="none"></i>
-        Upgrading</a>
-            </li>
-                                                                                                                                                      
-      <li>
-    
-                          <a href="../../metron-analytics/index.html" title="Analytics">
-          <i class="icon-chevron-right"></i>
-        Analytics</a>
-                  </li>
-                      
-      <li class="active">
-    
-            <a href="#"><i class="none"></i>Docker</a>
-          </li>
-                                                                                                                                                                                                                                                                                                                                                                                                            
-      <li>
-    
-                          <a href="../../metron-deployment/index.html" title="Deployment">
-          <i class="icon-chevron-right"></i>
-        Deployment</a>
-                  </li>
-                      
-      <li>
-    
-                          <a href="../../metron-interface/metron-alerts/index.html" title="Alerts">
-          <i class="none"></i>
-        Alerts</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-interface/metron-config/index.html" title="Config">
-          <i class="none"></i>
-        Config</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-interface/metron-rest/index.html" title="Rest">
-          <i class="none"></i>
-        Rest</a>
-            </li>
-                                                                                                                                                                                                                                                                                    
-      <li>
-    
-                          <a href="../../metron-platform/index.html" title="Platform">
-          <i class="icon-chevron-right"></i>
-        Platform</a>
-                  </li>
-                                                                                          
-      <li>
-    
-                          <a href="../../metron-sensors/index.html" title="Sensors">
-          <i class="icon-chevron-right"></i>
-        Sensors</a>
-                  </li>
-                      
-      <li>
-    
-                          <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example">
-          <i class="none"></i>
-        Stellar-3rd-party-example</a>
-            </li>
-                                                                        
-      <li>
-    
-                          <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common">
-          <i class="icon-chevron-right"></i>
-        Stellar-common</a>
-                  </li>
-                                                                                          
-      <li>
-    
-                          <a href="../../use-cases/index.html" title="Use-cases">
-          <i class="icon-chevron-right"></i>
-        Use-cases</a>
-                  </li>
-              </ul>
-        </li>
-            </ul>
-                
-                    
-                
-          <hr class="divider" />
-
-           <div id="poweredBy">
-                            <div class="clear"></div>
-                            <div class="clear"></div>
-                            <div class="clear"></div>
-                             <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
-        <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" />
-      </a>
-                  </div>
+    <ul class="nav nav-list">
+      <li class="nav-header">User Documentation</li>
+    <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+    <ul class="nav nav-list">
+    <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+    <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+    <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+    <li class="active"><a href="#"><span class="none"></span>Docker</a></li>
+    <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+    <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+    <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
+    <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
+    <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+    <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-right"></span>Platform</a></li>
+    <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+    <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+    <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+    <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+    <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+    </ul>
+</li>
+</ul>
+          <hr />
+          <div id="poweredBy">
+            <div class="clear"></div>
+            <div class="clear"></div>
+            <div class="clear"></div>
+            <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a>
+            </div>
           </div>
         </div>
-        
-                
-        <div id="bodyColumn"  class="span9" >
-                                  
-            <h1>Metron Docker</h1>
+        <div id="bodyColumn"  class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Metron Docker</h1>
 <p><a name="Metron_Docker"></a></p>
-<p>Metron Docker is a <a class="externalLink" href="https://docs.docker.com/compose/overview/">Docker Compose</a> application that is intended only for development and integration testing of Metron. These images can quickly spin-up the underlying components on which Apache Metron runs.</p>
-<p>None of the core Metron components are setup or launched automatically with these Docker images. You will need to manually setup and start the Metron components that you require. You should not expect to see telemetry being parsed, enriched, or indexed. If you are looking to try-out, experiment or demo Metron capabilities on a single node, then the <a href="../../metron-deployment/vagrant/full-dev-platform/index.html">Vagrant-driven VM</a> is what you need. Use this instead of Vagrant when:</p>
-
+<p>Metron Docker is a <a class="externalLink" href="https://docs.docker.com/compose/overview/">Docker Compose</a> application that is intended only for development and integration testing of Metron.  These images can quickly spin-up the underlying components on which Apache Metron runs.</p>
+<p>None of the core Metron components are setup or launched automatically with these Docker images.  You will need to manually setup and start the Metron components that you require.  You should not expect to see telemetry being parsed, enriched, or indexed.  If you are looking to try-out, experiment or demo Metron capabilities on a single node, then the <a href="../../metron-deployment/development/centos6/index.html">Vagrant-driven VM</a> is what you need.  Use this instead of Vagrant when:</p>
 <ul>
-  
+
 <li>You want an environment that can be built and spun up quickly</li>
-  
 <li>You need to frequently rebuild and restart services</li>
-  
 <li>You only need to test, troubleshoot or develop against a subset of services</li>
 </ul>
 <p>Metron Docker includes these images that have been customized for Metron:</p>
-
 <ul>
-  
+
 <li>Kafka (with Zookeeper)</li>
-  
 <li>HBase</li>
-  
 <li>Storm</li>
-  
 <li>Elasticsearch</li>
-  
 <li>Kibana</li>
-  
 <li>HDFS</li>
 </ul>
 <div class="section">
 <h2><a name="Setup"></a>Setup</h2>
-<p>Install <a class="externalLink" href="https://docs.docker.com/docker-for-mac/">Docker for Mac</a> or <a class="externalLink" href="https://docs.docker.com/docker-for-windows/">Docker for Windows</a>. The following versions have been tested:</p>
-
+<p>Install <a class="externalLink" href="https://docs.docker.com/docker-for-mac/">Docker for Mac</a> or <a class="externalLink" href="https://docs.docker.com/docker-for-windows/">Docker for Windows</a>.  The following versions have been tested:</p>
 <ul>
-  
+
 <li>Docker version 1.12.0</li>
-  
 <li>docker-machine version 0.8.0</li>
-  
 <li>docker-compose version 1.8.0</li>
 </ul>
 <p>Build Metron from the top level directory with:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ cd $METRON_HOME
+<div>
+<div>
+<pre class="source">$ cd $METRON_HOME
 $ mvn clean install -DskipTests
 </pre></div></div>
-<p>You are welcome to use an existing Docker host but we prefer one with more resources. You can create one of those with this script:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ export METRON_DOCKER_HOME=$METRON_HOME/metron-contrib/metron-docker
-$ cd $METRON_DOCKER_HOME 
+<p>You are welcome to use an existing Docker host but we prefer one with more resources.  You can create one of those with this script:</p>
+
+<div>
+<div>
+<pre class="source">$ export METRON_DOCKER_HOME=$METRON_HOME/metron-contrib/metron-docker
+$ cd $METRON_DOCKER_HOME
 $ ./scripts/create-docker-machine.sh
 </pre></div></div>
-<p>This will create a host called &#x201c;metron-machine&#x201d;. Anytime you want to run Docker commands against this host, make sure you run this first to set the Docker environment variables:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ eval &quot;$(docker-machine env metron-machine)&quot;
+<p>This will create a host called &#x201c;metron-machine&#x201d;.  Anytime you want to run Docker commands against this host, make sure you run this first to set the Docker environment variables:</p>
+
+<div>
+<div>
+<pre class="source">$ eval &quot;$(docker-machine env metron-machine)&quot;
 </pre></div></div>
+
 <p>If you wish to use a local docker-engine install, please set an environment variable BROKER_IP_ADDR to the IP address of your host machine. This cannot be the loopback address.</p></div>
 <div class="section">
 <h2><a name="Usage"></a>Usage</h2>
 <p>Navigate to the compose application root:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ cd $METRON_DOCKER_HOME/compose/
+<div>
+<div>
+<pre class="source">$ cd $METRON_DOCKER_HOME/compose/
 </pre></div></div>
-<p>The Metron Docker environment lifecycle is controlled by the <a class="externalLink" href="https://docs.docker.com/compose/reference/overview/">docker-compose</a> command. The service names can be found in the docker-compose.yml file. For example, to build and start the environment run this command:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ eval &quot;$(docker-machine env metron-machine)&quot;
+<p>The Metron Docker environment lifecycle is controlled by the <a class="externalLink" href="https://docs.docker.com/compose/reference/overview/">docker-compose</a> command.  The service names can be found in the docker-compose.yml file.  For example, to build and start the environment run this command:</p>
+
+<div>
+<div>
+<pre class="source">$ eval &quot;$(docker-machine env metron-machine)&quot;
 $ docker-compose up -d
 </pre></div></div>
+
 <p>After all services have started list the containers and ensure their status is &#x2018;Up&#x2019;:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ docker ps --format 'table {{.Names}}\t{{.Status}}'
+<div>
+<div>
+<pre class="source">$ docker ps --format 'table {{.Names}}\t{{.Status}}'
 NAMES                    STATUS
 metron_storm_1           Up 5 minutes
 metron_hbase_1           Up 5 minutes
@@ -276,216 +172,228 @@ metron_kibana_1          Up 5 minutes
 metron_kafkazk_1         Up 5 minutes
 metron_elasticsearch_1   Up 5 minutes
 </pre></div></div>
-<p>Various services are exposed through http on the Docker host. Get the host ip from the URL property:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ docker-machine ls
+<p>Various services are exposed through http on the Docker host.  Get the host ip from the URL property:</p>
+
+<div>
+<div>
+<pre class="source">$ docker-machine ls
 NAME             ACTIVE   DRIVER       STATE     URL                         SWARM   DOCKER    ERRORS
 metron-machine   *        virtualbox   Running   tcp://192.168.99.100:2376           v1.12.5
 </pre></div></div>
-<p>Then, assuming a host ip of <tt>192.168.99.100</tt>, the UIs and APIs are available at:</p>
 
+<p>Then, assuming a host ip of <tt>192.168.99.100</tt>, the UIs and APIs are available at:</p>
 <ul>
-  
+
 <li>Storm - <a class="externalLink" href="http://192.168.99.100:8080/">http://192.168.99.100:8080/</a></li>
-  
 <li>HBase - <a class="externalLink" href="http://192.168.99.100:16010/">http://192.168.99.100:16010/</a></li>
-  
 <li>Elasticsearch - <a class="externalLink" href="http://192.168.99.100:9200/_plugin/head/">http://192.168.99.100:9200/_plugin/head/</a></li>
-  
 <li>Kibana - <a class="externalLink" href="http://192.168.99.100:5601/">http://192.168.99.100:5601/</a></li>
-  
 <li>HDFS (Namenode) - <a class="externalLink" href="http://192.168.99.100:50070/">http://192.168.99.100:50070/</a></li>
 </ul>
-<p>The Storm logs can be useful when troubleshooting topologies. They can be found on the Storm container in <tt>/usr/share/apache-storm/logs</tt>.</p>
+<p>The Storm logs can be useful when troubleshooting topologies.  They can be found on the Storm container in <tt>/usr/share/apache-storm/logs</tt>.</p>
 <p>When done using the machine, shut it down with:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ docker-compose down
-</pre></div></div></div>
+<div>
+<div>
+<pre class="source">$ docker-compose down
+</pre></div></div>
+</div>
 <div class="section">
 <h2><a name="Examples"></a>Examples</h2>
-
 <ul>
-  
+
 <li><a href="#Deploy_a_new_parser_class">Deploy a new parser class</a></li>
-  
 <li><a href="#Connect_to_a_container">Connect to a container</a></li>
-  
 <li><a href="create-a-sensor-from-sample-data/index.html">Create a sensor from sample data</a></li>
-  
 <li><a href="upload-configs-to-zookeeper/index.html">Upload configs to Zookeeper</a></li>
-  
 <li><a href="manage-a-topology/index.html">Manage a topology</a></li>
-  
 <li><a href="run-sensor-data-end-to-end/index.html">Run sensor data end to end</a></li>
 </ul>
 <div class="section">
 <h3><a name="Deploy_a_new_parser_class"></a>Deploy a new parser class</h3>
 <p>After adding a new parser to metron-parsers, build Metron from the top level directory:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ cd $METRON_HOME
+<div>
+<div>
+<pre class="source">$ cd $METRON_HOME
 $ mvn clean install -DskipTests
 </pre></div></div>
+
 <p>Then run these commands to redeploy the parsers to the Storm image:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ cd $METRON_DOCKER_HOME/compose
+<div>
+<div>
+<pre class="source">$ cd $METRON_DOCKER_HOME/compose
 $ docker-compose down
 $ docker-compose build storm
 $ docker-compose up -d
-</pre></div></div></div>
+</pre></div></div>
+</div>
 <div class="section">
 <h3><a name="Connect_to_a_container"></a>Connect to a container</h3>
 <p>Suppose there is a problem with Kafka and the logs are needed for further investigation. Run this command to connect and explore the running Kafka container:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ cd $METRON_DOCKER_HOME/compose
+<div>
+<div>
+<pre class="source">$ cd $METRON_DOCKER_HOME/compose
 $ docker-compose exec kafkazk bash
-</pre></div></div></div>
+</pre></div></div>
+</div>
 <div class="section">
 <h3><a name="Create_a_sensor_from_sample_data"></a>Create a sensor from sample data</h3>
-<p>A tool for producing test data in Kafka is included with the Kafka/Zookeeper image. It loops through lines in a test data file and outputs them to Kafka at the desired frequency. Create a test data file in <tt>./kafkazk/data/</tt> and rebuild the Kafka/Zookeeper image:</p>
+<p>A tool for producing test data in Kafka is included with the Kafka/Zookeeper image.  It loops through lines in a test data file and outputs them to Kafka at the desired frequency.  Create a test data file in <tt>./kafkazk/data/</tt> and rebuild the Kafka/Zookeeper image:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ cd $METRON_DOCKER_HOME/compose
+<div>
+<div>
+<pre class="source">$ cd $METRON_DOCKER_HOME/compose
 $ printf 'first test data\nsecond test data\nthird test data\n' &gt; ./kafkazk/data/TestData.txt
 $ docker-compose down
 $ docker-compose build kafkazk
 $ docker-compose up -d
 </pre></div></div>
-<p>This will deploy the test data file to the Kafka/Zookeeper container. Now that data can be streamed to a Kafka topic:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ docker-compose exec kafkazk ./bin/produce-data.sh
+<p>This will deploy the test data file to the Kafka/Zookeeper container.  Now that data can be streamed to a Kafka topic:</p>
+
+<div>
+<div>
+<pre class="source">$ docker-compose exec kafkazk ./bin/produce-data.sh
 Usage:  produce-data.sh data_path topic [message_delay_in_seconds]
 
 # Stream data in TestData.txt to the 'test' Kafka topic at a frequency of 5 seconds (default is 1 second)
-$ docker-compose exec kafkazk ./bin/produce-data.sh /data/TestData.txt test 5 
+$ docker-compose exec kafkazk ./bin/produce-data.sh /data/TestData.txt test 5
 </pre></div></div>
+
 <p>The Kafka/Zookeeper image comes with sample Bro and Squid data:</p>
 
-<div class="source">
-<div class="source">
-<pre># Stream Bro test data every 1 second
+<div>
+<div>
+<pre class="source"># Stream Bro test data every 1 second
 $ docker-compose exec kafkazk ./bin/produce-data.sh /data/BroExampleOutput.txt bro
 
 # Stream Squid test data every 0.1 seconds
 $ docker-compose exec kafkazk ./bin/produce-data.sh /data/SquidExampleOutput.txt squid 0.1
-</pre></div></div></div>
+</pre></div></div>
+</div>
 <div class="section">
 <h3><a name="Upload_configs_to_Zookeeper"></a>Upload configs to Zookeeper</h3>
-<p>Parser configs and a global config configured for this Docker environment are included with the Kafka/Zookeeper image. Load them with:</p>
+<p>Parser configs and a global config configured for this Docker environment are included with the Kafka/Zookeeper image.  Load them with:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ docker-compose exec kafkazk bash
+<div>
+<div>
+<pre class="source">$ docker-compose exec kafkazk bash
 # $METRON_HOME/bin/zk_load_configs.sh -z localhost:2181 -m PUSH -i $METRON_HOME/config/zookeeper
 # exit
 </pre></div></div>
+
 <p>Dump out the configs with:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ docker-compose exec kafkazk bash
+<div>
+<div>
+<pre class="source">$ docker-compose exec kafkazk bash
 # $METRON_HOME/bin/zk_load_configs.sh -z localhost:2181 -m DUMP
 # exit
-</pre></div></div></div>
+</pre></div></div>
+</div>
 <div class="section">
 <h3><a name="Manage_a_topology"></a>Manage a topology</h3>
 <p>The Storm image comes with a script to easily start parser topologies:</p>
 
-<div class="source">
-<div class="source">
-<pre>docker-compose exec storm ./bin/start_docker_parser_topology.sh sensor_name
+<div>
+<div>
+<pre class="source">docker-compose exec storm ./bin/start_docker_parser_topology.sh sensor_name
 </pre></div></div>
+
 <p>The enrichment topology can be started with:</p>
 
-<div class="source">
-<div class="source">
-<pre>docker-compose exec storm ./bin/start_enrichment_topology.sh
+<div>
+<div>
+<pre class="source">docker-compose exec storm ./bin/start_enrichment_topology.sh
 </pre></div></div>
+
 <p>The indexing topology can be started with:</p>
 
-<div class="source">
-<div class="source">
-<pre>docker-compose exec storm ./bin/start_elasticsearch_topology.sh
+<div>
+<div>
+<pre class="source">docker-compose exec storm ./bin/start_elasticsearch_topology.sh
 </pre></div></div>
-<p>Topologies can be stopped using the Storm CLI. For example, stop the enrichment topology with:</p>
 
-<div class="source">
-<div class="source">
-<pre>docker-compose exec storm storm kill enrichments -w 0
-</pre></div></div></div>
+<p>Topologies can be stopped using the Storm CLI.  For example, stop the enrichment topology with:</p>
+
+<div>
+<div>
+<pre class="source">docker-compose exec storm storm kill enrichments -w 0
+</pre></div></div>
+</div>
 <div class="section">
 <h3><a name="Run_sensor_data_end_to_end"></a>Run sensor data end to end</h3>
 <p>First ensure configs were uploaded as described in the previous example. Then start a sensor and leave it running:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ cd $METRON_DOCKER_HOME/compose
+<div>
+<div>
+<pre class="source">$ cd $METRON_DOCKER_HOME/compose
 $ docker-compose exec kafkazk ./bin/produce-data.sh /data/BroExampleOutput.txt bro
 </pre></div></div>
+
 <p>Open a separate console session and verify the sensor is running by consuming a message from Kafka:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ export METRON_DOCKER_HOME=$METRON_HOME/metron-contrib/metron-docker
+<div>
+<div>
+<pre class="source">$ export METRON_DOCKER_HOME=$METRON_HOME/metron-contrib/metron-docker
 $ cd $METRON_DOCKER_HOME/compose
 $ docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic bro
 </pre></div></div>
+
 <p>A new message should be printed every second. Now kill the consumer and start the Bro parser topology:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ docker-compose exec storm ./bin/start_docker_parser_topology.sh bro
+<div>
+<div>
+<pre class="source">$ docker-compose exec storm ./bin/start_docker_parser_topology.sh bro
 </pre></div></div>
-<p>Bro data should be flowing through the bro parser topology and into the Kafka enrichments topic. The enrichments topic should be created automatically:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ docker-compose exec kafkazk ./bin/kafka-topics.sh --zookeeper localhost:2181 --list
+<p>Bro data should be flowing through the bro parser topology and into the Kafka enrichments topic.  The enrichments topic should be created automatically:</p>
+
+<div>
+<div>
+<pre class="source">$ docker-compose exec kafkazk ./bin/kafka-topics.sh --zookeeper localhost:2181 --list
 bro
 enrichments
 indexing
 </pre></div></div>
+
 <p>Verify parsed Bro data is in the Kafka enrichments topic:</p>
 
-<div class="source">
-<div class="source">
-<pre>docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic enrichments
+<div>
+<div>
+<pre class="source">docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic enrichments
 </pre></div></div>
+
 <p>Now start the enrichment topology:</p>
 
-<div class="source">
-<div class="source">
-<pre>docker-compose exec storm ./bin/start_enrichment_topology.sh
+<div>
+<div>
+<pre class="source">docker-compose exec storm ./bin/start_enrichment_topology.sh
 </pre></div></div>
-<p>Parsed Bro data should be flowing through the enrichment topology and into the Kafka indexing topic. Verify enriched Bro data is in the Kafka indexing topic:</p>
 
-<div class="source">
-<div class="source">
-<pre>docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic indexing
+<p>Parsed Bro data should be flowing through the enrichment topology and into the Kafka indexing topic.  Verify enriched Bro data is in the Kafka indexing topic:</p>
+
+<div>
+<div>
+<pre class="source">docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic indexing
 </pre></div></div>
+
 <p>Now start the indexing topology:</p>
 
-<div class="source">
-<div class="source">
-<pre>docker-compose exec storm ./bin/start_elasticsearch_topology.sh
+<div>
+<div>
+<pre class="source">docker-compose exec storm ./bin/start_elasticsearch_topology.sh
 </pre></div></div>
+
 <p>Enriched Bro data should now be present in the Elasticsearch container:</p>
 
-<div class="source">
-<div class="source">
-<pre>$ docker-machine ls
+<div>
+<div>
+<pre class="source">$ docker-machine ls
 NAME             ACTIVE   DRIVER       STATE     URL                         SWARM   DOCKER    ERRORS
 metron-machine   *        virtualbox   Running   tcp://192.168.99.100:2376           v1.12.5
 
@@ -494,23 +402,17 @@ health status index                   pri rep docs.count docs.deleted store.size
 yellow open   .kibana                   1   1          1            0      3.1kb          3.1kb
 yellow open   bro_index_2016.12.19.18   5   1        180            0      475kb          475kb
 </pre></div></div></div></div>
-                  </div>
-            </div>
-          </div>
-
+        </div>
+      </div>
+    </div>
     <hr/>
-
     <footer>
-            <div class="container-fluid">
-              <div class="row span12">Copyright &copy;                    2018
-                        <a href="https://www.apache.org">The Apache Software Foundation</a>.
-            All Rights Reserved.      
-                    
+      <div class="container-fluid">
+        <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+            and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+        </div>
       </div>
-
-                          
-        
-                </div>
     </footer>
   </body>
 </html>


Mime
View raw message