metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject [08/34] metron git commit: METRON-1607: update public web site to point at 0.5.0 new release
Date Fri, 08 Jun 2018 12:42:39 GMT
http://git-wip-us.apache.org/repos/asf/metron/blob/8b8505da/current-book/metron-platform/metron-management/index.html
----------------------------------------------------------------------
diff --git a/current-book/metron-platform/metron-management/index.html b/current-book/metron-platform/metron-management/index.html
index cca14f1..f9ea1ce 100644
--- a/current-book/metron-platform/metron-management/index.html
+++ b/current-book/metron-platform/metron-management/index.html
@@ -1,1040 +1,644 @@
 <!DOCTYPE html>
 <!--
- | Generated by Apache Maven Doxia at 2018-01-03
- | Rendered using Apache Maven Fluido Skin 1.3.0
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-management/index.md at 2018-06-07
+ | Rendered using Apache Maven Fluido Skin 1.7
 -->
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <meta name="Date-Revision-yyyymmdd" content="20180103" />
+    <meta name="Date-Revision-yyyymmdd" content="20180607" />
     <meta http-equiv="Content-Language" content="en" />
     <title>Metron &#x2013; Stellar REPL Management Utilities</title>
-    <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
+    <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
     <link rel="stylesheet" href="../../css/site.css" />
     <link rel="stylesheet" href="../../css/print.css" media="print" />
-
-      
-    <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script>
-
-                          
-        
-<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
-          
-            </head>
-        <body class="topBarDisabled">
-          
-                
-                    
-    
-        <div class="container-fluid">
-          <div id="banner">
-        <div class="pull-left">
-                                    <a href="http://metron.apache.org/" id="bannerLeft">
-                                                                                                <img src="../../images/metron-logo.png"  alt="Apache Metron" width="148px" height="48px"/>
-                </a>
-                      </div>
-        <div class="pull-right">  </div>
+    <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+              $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+            </script>
+  </head>
+  <body class="topBarDisabled">
+    <div class="container-fluid">
+      <div id="banner">
+        <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../images/metron-logo.png"  alt="Apache Metron" width="148px" height="48px"/></a></div>
+        <div class="pull-right"></div>
         <div class="clear"><hr/></div>
       </div>
 
       <div id="breadcrumbs">
         <ul class="breadcrumb">
-                
-                    
-                              <li class="">
-                    <a href="http://www.apache.org" class="externalLink" title="Apache">
-        Apache</a>
-        </li>
-      <li class="divider ">/</li>
-            <li class="">
-                    <a href="http://metron.apache.org/" class="externalLink" title="Metron">
-        Metron</a>
-        </li>
-      <li class="divider ">/</li>
-            <li class="">
-                    <a href="../../index.html" title="Documentation">
-        Documentation</a>
-        </li>
-      <li class="divider ">/</li>
-        <li class="">Stellar REPL Management Utilities</li>
-        
-                
-                    
-                  <li id="publishDate" class="pull-right">Last Published: 2018-01-03</li> <li class="divider pull-right">|</li>
-              <li id="projectVersion" class="pull-right">Version: 0.4.2</li>
-            
-                            </ul>
+      <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+      <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+      <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+    <li class="active ">Stellar REPL Management Utilities</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
+          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        </ul>
       </div>
-
-            
       <div class="row-fluid">
-        <div id="leftColumn" class="span3">
+        <div id="leftColumn" class="span2">
           <div class="well sidebar-nav">
-                
-                    
-                <ul class="nav nav-list">
-                    <li class="nav-header">User Documentation</li>
-                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     
                                                                          
-      <li>
-    
-                          <a href="../../index.html" title="Metron">
-          <i class="icon-chevron-down"></i>
-        Metron</a>
-                    <ul class="nav nav-list">
-                      
-      <li>
-    
-                          <a href="../../Upgrading.html" title="Upgrading">
-          <i class="none"></i>
-        Upgrading</a>
-            </li>
-                                                                                                                                                      
-      <li>
-    
-                          <a href="../../metron-analytics/index.html" title="Analytics">
-          <i class="icon-chevron-right"></i>
-        Analytics</a>
-                  </li>
-                      
-      <li>
-    
-                          <a href="../../metron-contrib/metron-docker/index.html" title="Docker">
-          <i class="none"></i>
-        Docker</a>
-            </li>
-                                                                                                                                                                                                                                                                                                                                                                                                            
-      <li>
-    
-                          <a href="../../metron-deployment/index.html" title="Deployment">
-          <i class="icon-chevron-right"></i>
-        Deployment</a>
-                  </li>
-                      
-      <li>
-    
-                          <a href="../../metron-interface/metron-alerts/index.html" title="Alerts">
-          <i class="none"></i>
-        Alerts</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-interface/metron-config/index.html" title="Config">
-          <i class="none"></i>
-        Config</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-interface/metron-rest/index.html" title="Rest">
-          <i class="none"></i>
-        Rest</a>
-            </li>
-                                                                                                                                                                                                                                                                                              
-      <li>
-    
-                          <a href="../../metron-platform/index.html" title="Platform">
-          <i class="icon-chevron-down"></i>
-        Platform</a>
-                    <ul class="nav nav-list">
-                      
-      <li>
-    
-                          <a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide">
-          <i class="none"></i>
-        Performance-tuning-guide</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-platform/metron-api/index.html" title="Api">
-          <i class="none"></i>
-        Api</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-platform/metron-common/index.html" title="Common">
-          <i class="none"></i>
-        Common</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-platform/metron-data-management/index.html" title="Data-management">
-          <i class="none"></i>
-        Data-management</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch">
-          <i class="none"></i>
-        Elasticsearch</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment">
-          <i class="none"></i>
-        Enrichment</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-platform/metron-indexing/index.html" title="Indexing">
-          <i class="none"></i>
-        Indexing</a>
-            </li>
-                      
-      <li class="active">
-    
-            <a href="#"><i class="none"></i>Management</a>
-          </li>
-                                                                        
-      <li>
-    
-                          <a href="../../metron-platform/metron-parsers/index.html" title="Parsers">
-          <i class="icon-chevron-right"></i>
-        Parsers</a>
-                  </li>
-                      
-      <li>
-    
-                          <a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend">
-          <i class="none"></i>
-        Pcap-backend</a>
-            </li>
-                      
-      <li>
-    
-                          <a href="../../metron-platform/metron-writer/index.html" title="Writer">
-          <i class="none"></i>
-        Writer</a>
-            </li>
-              </ul>
-        </li>
-                                                                                          
-      <li>
-    
-                          <a href="../../metron-sensors/index.html" title="Sensors">
-          <i class="icon-chevron-right"></i>
-        Sensors</a>
-                  </li>
-                      
-      <li>
-    
-                          <a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example">
-          <i class="none"></i>
-        Stellar-3rd-party-example</a>
-            </li>
-                                                                        
-      <li>
-    
-                          <a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common">
-          <i class="icon-chevron-right"></i>
-        Stellar-common</a>
-                  </li>
-                                                                                          
-      <li>
-    
-                          <a href="../../use-cases/index.html" title="Use-cases">
-          <i class="icon-chevron-right"></i>
-        Use-cases</a>
-                  </li>
-              </ul>
-        </li>
-            </ul>
-                
-                    
-                
-          <hr class="divider" />
-
-           <div id="poweredBy">
-                            <div class="clear"></div>
-                            <div class="clear"></div>
-                            <div class="clear"></div>
-                             <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
-        <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" />
-      </a>
-                  </div>
+    <ul class="nav nav-list">
+      <li class="nav-header">User Documentation</li>
+    <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+    <ul class="nav nav-list">
+    <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+    <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+    <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+    <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+    <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+    <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+    <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
+    <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
+    <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+    <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+    <ul class="nav nav-list">
+    <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+    <li><a href="../../metron-platform/metron-api/index.html" title="Api"><span class="none"></span>Api</a></li>
+    <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+    <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+    <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+    <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+    <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+    <li class="active"><a href="#"><span class="none"></span>Management</a></li>
+    <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-right"></span>Parsers</a></li>
+    <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+    <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+    </ul>
+</li>
+    <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+    <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+    <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+    <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+    <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+    </ul>
+</li>
+</ul>
+          <hr />
+          <div id="poweredBy">
+            <div class="clear"></div>
+            <div class="clear"></div>
+            <div class="clear"></div>
+            <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a>
+            </div>
           </div>
         </div>
-        
-                
-        <div id="bodyColumn"  class="span9" >
-                                  
-            <h1>Stellar REPL Management Utilities</h1>
+        <div id="bodyColumn"  class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Stellar REPL Management Utilities</h1>
 <p><a name="Stellar_REPL_Management_Utilities"></a></p>
 <p>In order to augment the functionality of the Stellar REPL, a few management functions surrounding the management of the configurations and the management of Stellar transformations in the following areas have been added:</p>
-
 <ul>
-  
+
 <li>Stellar field transformations in the Parsers</li>
-  
 <li>Stellar enrichments in the Enrichment topology</li>
-  
 <li>Stellar threat triage rules</li>
 </ul>
-<p>Additionally, some shell functions have been added to </p>
-
+<p>Additionally, some shell functions have been added to</p>
 <ul>
-  
+
 <li>provide the ability to refer to the Stellar expression used to create a variable</li>
-  
 <li>print structured data in a way that is easier to view (i.e. tabular)</li>
 </ul>
 <p>This functionality is exposed as a pack of Stellar functions in this project.</p>
-
 <ul>
-  
+
 <li><a href="#Functions">Functions</a>
-  
 <ul>
-    
+
 <li><a href="#Grok_Functions">Grok Functions</a></li>
-    
 <li><a href="#File_Functions">File Functions</a></li>
-    
-<li><a href="#Shell_Functions">Shell Functions</a></li>
-    
 <li><a href="#Configuration_Functions">Configuration Functions</a></li>
-    
 <li><a href="#Parser_Functions">Parser Functions</a></li>
-    
 <li><a href="#Indexing_Functions">Indexing Functions</a></li>
-    
 <li><a href="#Enrichment_Functions">Enrichment Functions</a></li>
-    
 <li><a href="#Threat_Triage_Functions">Threat Triage Functions</a></li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><a href="#Examples">Examples</a>
-  
 <ul>
-    
+
 <li><a href="#Iterate_to_Find_a_Valid_Grok_Pattern">Iterate to Find a Valid Grok Pattern</a></li>
-    
 <li><a href="#Manage_Stellar_Field_Transformations">Manage Stellar Field Transformations</a></li>
-    
 <li><a href="#Manage_Stellar_Enrichments">Manage Stellar Enrichments</a></li>
-    
 <li><a href="#Manage_Threat_Triage_Rules">Manage Threat Triage Rules</a></li>
-    
 <li><a href="#Simulate_Threat_Triage_Rules">Simulate Threat Triage Rules</a></li>
-  </ul></li>
+</ul>
+</li>
 </ul>
 <div class="section">
 <h2><a name="Functions"></a>Functions</h2>
 <p>The functions are split roughly into a few sections:</p>
-
 <ul>
-  
+
 <li>Shell functions - Functions surrounding interacting with the shell in either a nicer way or a more functional way.</li>
-  
 <li>Grok Functions - Functions that allow you to evaluate grok expressions.</li>
-  
 <li>File functions - Functions around interacting with local or HDFS files</li>
-  
 <li>Configuration functions - Functions surrounding pulling and pushing configs from zookeeper</li>
-  
 <li>Parser functions - Functions surrounding adding, viewing, and removing Parser functions.</li>
-  
 <li>Enrichment functions - Functions surrounding adding, viewing and removing Stellar enrichments as well as managing batch size, batch timeout, and index names for the enrichment topology configuration</li>
-  
 <li>Threat Triage functions - Functions surrounding adding, viewing and removing threat triage functions.</li>
 </ul>
 <div class="section">
 <h3><a name="Grok_Functions"></a>Grok Functions</h3>
-
 <ul>
-  
+
 <li><tt>GROK_EVAL</tt>
-  
 <ul>
-    
+
 <li>Description: Evaluate a grok expression for a statement.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>grokExpression - The grok expression to evaluate</li>
-      
 <li>data - Either a data message or a list of data messages to evaluate using the grokExpression</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The Map associated with the grok expression being evaluated on the list of messages.</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>GROK_PREDICT</tt>
-  
 <ul>
-    
+
 <li>Description: Discover a grok statement for an input doc</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>data - The data to discover a grok expression from</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: A grok expression that should match the data.</li>
-  </ul></li>
+</ul>
+</li>
 </ul></div>
 <div class="section">
 <h3><a name="File_Functions"></a>File Functions</h3>
-
 <ul>
-  
+
 <li>Local Files
-  
 <ul>
-    
+
 <li><tt>LOCAL_LS</tt>
-    
 <ul>
-      
+
 <li>Description: Lists the contents of a directory.</li>
-      
 <li>Input:
-      
 <ul>
-        
+
 <li>path - The path of the file</li>
-      </ul></li>
-      
+</ul>
+</li>
 <li>Returns: The contents of the directory in tabular form sorted by last modification date.</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li><tt>LOCAL_RM</tt>
-    
 <ul>
-      
+
 <li>Description: Removes the path</li>
-      
 <li>Input:
-      
 <ul>
-        
+
 <li>path - The path of the file or directory.</li>
-        
 <li>recursive - Recursively remove or not (optional and defaulted to false)</li>
-      </ul></li>
-      
+</ul>
+</li>
 <li>Returns: boolean - true if successful, false otherwise</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li><tt>LOCAL_READ</tt>
-    
 <ul>
-      
+
 <li>Description: Retrieves the contents as a string of a file.</li>
-      
 <li>Input:
-      
 <ul>
-        
+
 <li>path - The path of the file</li>
-      </ul></li>
-      
+</ul>
+</li>
 <li>Returns: The contents of the file and null otherwise.</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li><tt>LOCAL_READ_LINES</tt>
-    
 <ul>
-      
+
 <li>Description: Retrieves the contents of a file as a list of strings.</li>
-      
 <li>Input:
-      
 <ul>
-        
+
 <li>path - The path of the file</li>
-      </ul></li>
-      
+</ul>
+</li>
 <li>Returns: A list of lines</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li><tt>LOCAL_WRITE</tt>
-    
 <ul>
-      
+
 <li>Description: Writes the contents of a string to a local file</li>
-      
 <li>Input:
-      
 <ul>
-        
+
 <li>content - The content to write out</li>
-        
 <li>path - The path of the file</li>
-      </ul></li>
-      
+</ul>
+</li>
 <li>Returns: true if the file was written and false otherwise.</li>
-    </ul></li>
-  </ul></li>
-  
+</ul>
+</li>
+</ul>
+</li>
 <li>HDFS Files
-  
 <ul>
-    
+
 <li><tt>HDFS_LS</tt>
-    
 <ul>
-      
+
 <li>Description: Lists the contents of a directory in HDFS.</li>
-      
 <li>Input:
-      
 <ul>
-        
+
 <li>path - The path of the file</li>
-      </ul></li>
-      
+</ul>
+</li>
 <li>Returns: The contents of the directory in tabular form sorted by last modification date.</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li><tt>HDFS_RM</tt>
-    
 <ul>
-      
+
 <li>Description: Removes the path in HDFS.</li>
-      
 <li>Input:
-      
 <ul>
-        
+
 <li>path - The path of the file or directory.</li>
-        
 <li>recursive - Recursively remove or not (optional and defaulted to false)</li>
-      </ul></li>
-      
+</ul>
+</li>
 <li>Returns: boolean - true if successful, false otherwise</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li><tt>HDFS_READ</tt>
-    
 <ul>
-      
+
 <li>Description: Retrieves the contents as a string of a file in HDFS.</li>
-      
 <li>Input:
-      
 <ul>
-        
+
 <li>path - The path of the file</li>
-      </ul></li>
-      
+</ul>
+</li>
 <li>Returns: The contents of the file and null otherwise.</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li><tt>HDFS_READ_LINES</tt>
-    
 <ul>
-      
+
 <li>Description: Retrieves the contents of a HDFS file as a list of strings.</li>
-      
 <li>Input:
-      
 <ul>
-        
+
 <li>path - The path of the file</li>
-      </ul></li>
-      
+</ul>
+</li>
 <li>Returns: A list of lines</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li><tt>HDFS_WRITE</tt>
-    
 <ul>
-      
+
 <li>Description: Writes the contents of a string to a HDFS file</li>
-      
 <li>Input:
-      
 <ul>
-        
+
 <li>content - The content to write out</li>
-        
 <li>path - The path of the file</li>
-      </ul></li>
-      
+</ul>
+</li>
 <li>Returns: true if the file was written and false otherwise.</li>
-    </ul></li>
-  </ul></li>
-</ul></div>
-<div class="section">
-<h3><a name="Shell_Functions"></a>Shell Functions</h3>
-
-<ul>
-  
-<li><tt>SHELL_EDIT</tt>
-  
-<ul>
-    
-<li>Description: Open an editor (optionally initialized with text) and return whatever is saved from the editor. The editor to use is pulled from <tt>EDITOR</tt> or <tt>VISUAL</tt> environment variable.</li>
-    
-<li>Input:
-    
-<ul>
-      
-<li>string - (Optional) A string whose content is used to initialize the editor.</li>
-    </ul></li>
-    
-<li>Returns: The content that the editor saved after editor exit.</li>
-  </ul></li>
-  
-<li><tt>SHELL_GET_EXPRESSION</tt>
-  
-<ul>
-    
-<li>Description: Get a stellar expression from a variable</li>
-    
-<li>Input:
-    
-<ul>
-      
-<li>variable - variable name</li>
-    </ul></li>
-    
-<li>Returns: The stellar expression associated with the variable.</li>
-  </ul></li>
-  
-<li><tt>SHELL_LIST_VARS</tt>
-  
-<ul>
-    
-<li>Description: Return the variables in a tabular form</li>
-    
-<li>Input:
-    
-<ul>
-      
-<li>wrap : Length of string to wrap the columns</li>
-    </ul></li>
-    
-<li>Returns: A tabular representation of the variables.</li>
-  </ul></li>
-  
-<li><tt>SHELL_MAP2TABLE</tt>
-  
-<ul>
-    
-<li>Description: Take a map and return a table</li>
-    
-<li>Input:
-    
-<ul>
-      
-<li>map - Map</li>
-    </ul></li>
-    
-<li>Returns: The map in table form</li>
-  </ul></li>
-  
-<li><tt>SHELL_VARS2MAP</tt>
-  
-<ul>
-    
-<li>Description: Take a set of variables and return a map</li>
-    
-<li>Input:
-    
-<ul>
-      
-<li>variables* - variable names to use to create map</li>
-    </ul></li>
-    
-<li>Returns: A map associating the variable name with the stellar expression.</li>
-  </ul></li>
+</ul>
+</li>
+</ul>
+</li>
 </ul></div>
 <div class="section">
 <h3><a name="Configuration_Functions"></a>Configuration Functions</h3>
-
 <ul>
-  
+
 <li><tt>CONFIG_GET</tt>
-  
 <ul>
-    
+
 <li>Description: Retrieve a Metron configuration from zookeeper.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>type - One of ENRICHMENT, INDEXING, PARSER, GLOBAL, PROFILER</li>
-      
 <li>sensor - Sensor to retrieve (required for enrichment and parser, not used for profiler and global)</li>
-      
 <li>emptyIfNotPresent - If true, then return an empty, minimally viable config</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the config in zookeeper</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>CONFIG_PUT</tt>
-  
 <ul>
-    
+
 <li>Description: Updates a Metron config to Zookeeper.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>type - One of ENRICHMENT, INDEXING, PARSER, GLOBAL, PROFILER</li>
-      
 <li>config - The config (a string in JSON form) to update</li>
-      
 <li>sensor - Sensor to retrieve (required for enrichment and parser, not used for profiler and global)</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the config in zookeeper</li>
-  </ul></li>
+</ul>
+</li>
 </ul></div>
 <div class="section">
 <h3><a name="Parser_Functions"></a>Parser Functions</h3>
-
 <ul>
-  
+
 <li><tt>PARSER_STELLAR_TRANSFORM_ADD</tt>
-  
 <ul>
-    
+
 <li>Description: Add stellar field transformation.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-      
 <li>stellarTransforms - A Map associating fields to stellar expressions</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the config in zookeeper</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>PARSER-STELLAR_TRANSFORM_PRINT</tt>
-  
 <ul>
-    
+
 <li>Description: Retrieve stellar field transformations.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the transformations</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>PARSER_STELLAR_TRANSFORM_REMOVE</tt>
-  
 <ul>
-    
+
 <li>Description: Remove stellar field transformation.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-      
 <li>stellarTransforms - A list of stellar transforms to remove</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the config in zookeeper</li>
-  </ul></li>
+</ul>
+</li>
 </ul></div>
 <div class="section">
 <h3><a name="Indexing_Functions"></a>Indexing Functions</h3>
-
 <ul>
-  
+
 <li><tt>INDEXING_SET_BATCH</tt>
-  
 <ul>
-    
+
 <li>Description: Set batch size and timeout</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-      
 <li>writer - The writer to update (e.g. elasticsearch, solr or hdfs)</li>
-      
 <li>size - batch size (integer), defaults to 1, meaning batching disabled</li>
-      
 <li>timeout - (optional) batch timeout in seconds (integer), defaults to 0, meaning system default</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the config in zookeeper</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>INDEXING_SET_ENABLED</tt>
-  
 <ul>
-    
+
 <li>Description: Enable or disable an indexing writer for a sensor.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-      
 <li>writer - The writer to update (e.g. elasticsearch, solr or hdfs)</li>
-      
-<li>enabled? - boolean indicating whether the writer is enabled. If omitted, then it will set enabled.</li>
-    </ul></li>
-    
+<li>enabled? - boolean indicating whether the writer is enabled.  If omitted, then it will set enabled.</li>
+</ul>
+</li>
 <li>Returns: The String representation of the config in zookeeper</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>INDEXING_SET_INDEX</tt>
-  
 <ul>
-    
+
 <li>Description: Set the index for the sensor</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-      
 <li>writer - The writer to update (e.g. elasticsearch, solr or hdfs)</li>
-      
 <li>sensor - sensor name</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the config in zookeeper</li>
-  </ul></li>
+</ul>
+</li>
 </ul></div>
 <div class="section">
 <h3><a name="Enrichment_Functions"></a>Enrichment Functions</h3>
-
 <ul>
-  
+
 <li><tt>ENRICHMENT_STELLAR_TRANSFORM_ADD</tt>
-  
 <ul>
-    
+
 <li>Description: Add stellar field transformation.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-      
 <li>type - ENRICHMENT or THREAT_INTEL</li>
-      
 <li>stellarTransforms - A Map associating fields to stellar expressions</li>
-      
 <li>group - Group to add to (optional)</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the config in zookeeper</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>ENRICHMENT_STELLAR_TRANSFORM_PRINT</tt>
-  
 <ul>
-    
+
 <li>Description: Retrieve stellar enrichment transformations.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-      
 <li>type - ENRICHMENT or THREAT_INTEL</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the transformations</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>ENRICHMENT_STELLAR_TRANSFORM_REMOVE</tt>
-  
 <ul>
-    
+
 <li>Description: Remove one or more stellar field transformations.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-      
 <li>type - ENRICHMENT or THREAT_INTEL</li>
-      
 <li>stellarTransforms - A list of removals</li>
-      
 <li>group - Group to remove from (optional)</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the config in zookeeper</li>
-  </ul></li>
+</ul>
+</li>
 </ul></div>
 <div class="section">
 <h3><a name="Threat_Triage_Functions"></a>Threat Triage Functions</h3>
-
 <ul>
-  
+
 <li><tt>THREAT_TRIAGE_INIT</tt>
-  
 <ul>
-    
+
 <li>Description: Create a threat triage engine.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>config - the threat triage configuration (optional)</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: A threat triage engine.</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>THREAT_TRIAGE_CONFIG</tt>
-  
 <ul>
-    
+
 <li>Description: Export the configuration used by a threat triage engine.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>engine - threat triage engine returned by THREAT_TRIAGE_INIT.</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The configuration used by the threat triage engine.</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>THREAT_TRIAGE_SCORE</tt>
-  
 <ul>
-    
+
 <li>Description: Scores a message using a set of triage rules.</li>
-    
 <li>Inputs:
-    
 <ul>
-      
+
 <li>message - a string containing the message to score.</li>
-      
 <li>engine - threat triage engine returned by THREAT_TRIAGE_INIT.</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: A threat triage engine.</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>THREAT_TRIAGE_ADD</tt>
-  
 <ul>
-    
+
 <li>Description: Add a threat triage rule.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-      
 <li>stellarTransforms - A Map associating stellar rules to scores</li>
-      
-<li>triageRules - Map (or list of Maps) representing a triage rule. It must contain &#x2018;rule&#x2019; and &#x2018;score&#x2019; keys, the stellar expression for the rule and triage score respectively. It may contain &#x2018;name&#x2019; and &#x2018;comment&#x2019;, the name of the rule and comment associated with the rule respectively.&quot;</li>
-    </ul></li>
-    
+<li>triageRules - Map (or list of Maps) representing a triage rule.  It must contain &#x2018;rule&#x2019; and &#x2018;score&#x2019; keys, the stellar expression for the rule and triage score respectively.  It may contain &#x2018;name&#x2019; and &#x2018;comment&#x2019;, the name of the rule and comment associated with the rule respectively.&quot;</li>
+</ul>
+</li>
 <li>Returns: The String representation of the threat triage rules</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>THREAT_TRIAGE_REMOVE</tt>
-  
 <ul>
-    
+
 <li>Description: Remove stellar threat triage rule(s).</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-      
 <li>rules - A list of stellar rules or rule names to remove</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the enrichment config</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>THREAT_TRIAGE_PRINT</tt>
-  
 <ul>
-    
+
 <li>Description: Retrieve stellar enrichment transformations.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the threat triage rules</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li><tt>THREAT_TRIAGE_SET_AGGREGATOR</tt>
-  
 <ul>
-    
+
 <li>Description: Set the threat triage aggregator.</li>
-    
 <li>Input:
-    
 <ul>
-      
+
 <li>sensorConfig - Sensor config to add transformation to.</li>
-      
-<li>aggregator - Aggregator to use. One of MIN, MAX, MEAN, SUM, POSITIVE_MEAN</li>
-      
+<li>aggregator - Aggregator to use.  One of MIN, MAX, MEAN, SUM, POSITIVE_MEAN</li>
 <li>aggregatorConfig - Optional config for aggregator</li>
-    </ul></li>
-    
+</ul>
+</li>
 <li>Returns: The String representation of the enrichment config</li>
-  </ul></li>
+</ul>
+</li>
 </ul></div></div>
 <div class="section">
 <h2><a name="Deployment_Instructions"></a>Deployment Instructions</h2>
-
 <ul>
-  
+
 <li>Clusters installed via Ambari Management Pack (default)
-  
 <ul>
-    
+
 <li>Automatically deployed</li>
-  </ul></li>
-  
+</ul>
+</li>
 <li>Manual installation
-  
 <ul>
-    
+
 <li>Deployment is as simple as dropping the jar created by this project into <tt>$METRON_HOME/lib</tt> and starting the Stellar shell via <tt>$METRON_HOME/bin/stellar</tt></li>
-  </ul></li>
+</ul>
+</li>
 </ul></div>
 <div class="section">
 <h2><a name="Examples"></a>Examples</h2>
@@ -1042,9 +646,9 @@
 <div class="section">
 <h3><a name="Iterate_to_Find_a_Valid_Grok_pattern"></a>Iterate to Find a Valid Grok pattern</h3>
 
-<div class="source">
-<div class="source">
-<pre>Stellar, Go!
+<div>
+<div>
+<pre class="source">Stellar, Go!
 Please note that functions are loading lazily in the background and will be unavailable until loaded fully.
 [Stellar]&gt;&gt;&gt; # We are going to debug a squid grok statement with a bug in it
 [Stellar]&gt;&gt;&gt; squid_grok_orig := '%{NUMBER:timestamp} %{SPACE:UNWANTED}  %{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} 
@@ -1117,13 +721,14 @@ WORD:UNWANTED}/(%{IP:ip_dst_addr})?'
 [Stellar]&gt;&gt;&gt; # Ahh, that is much better.
 [Stellar]&gt;&gt;&gt; 
 [Stellar]&gt;&gt;&gt; 
-</pre></div></div></div>
+</pre></div></div>
+</div>
 <div class="section">
 <h3><a name="Manage_Stellar_Field_Transformations"></a>Manage Stellar Field Transformations</h3>
 
-<div class="source">
-<div class="source">
-<pre>964  [main] INFO  o.a.c.f.i.CuratorFrameworkImpl - Starting
+<div>
+<div>
+<pre class="source">964  [main] INFO  o.a.c.f.i.CuratorFrameworkImpl - Starting
 1025 [main-EventThread] INFO  o.a.c.f.s.ConnectionStateManager - State change: CONNECTED
 Stellar, Go!
 Please note that functions are loading lazily in the background and will be unavailable until loaded fully.
@@ -1304,13 +909,14 @@ Returns: The String representation of the config in zookeeper
 }
 [Stellar]&gt;&gt;&gt; #And quit the REPL
 [Stellar]&gt;&gt;&gt; quit
-</pre></div></div></div>
+</pre></div></div>
+</div>
 <div class="section">
 <h3><a name="Manage_Stellar_Enrichments"></a>Manage Stellar Enrichments</h3>
 
-<div class="source">
-<div class="source">
-<pre>1010 [main] INFO  o.a.c.f.i.CuratorFrameworkImpl - Starting
+<div>
+<div>
+<pre class="source">1010 [main] INFO  o.a.c.f.i.CuratorFrameworkImpl - Starting
 1077 [main-EventThread] INFO  o.a.c.f.s.ConnectionStateManager - State change: CONNECTED
 Stellar, Go!
 Please note that functions are loading lazily in the background and will be unavailable until loaded fully.
@@ -1540,13 +1146,14 @@ Returns: The String representation of the config in zookeeper
   &quot;configuration&quot; : { }
 }
 [Stellar]&gt;&gt;&gt; 
-</pre></div></div></div>
+</pre></div></div>
+</div>
 <div class="section">
 <h3><a name="Manage_Threat_Triage_Rules"></a>Manage Threat Triage Rules</h3>
 
-<div class="source">
-<div class="source">
-<pre>987  [main] INFO  o.a.c.f.i.CuratorFrameworkImpl - Starting
+<div>
+<div>
+<pre class="source">987  [main] INFO  o.a.c.f.i.CuratorFrameworkImpl - Starting
 1047 [main-EventThread] INFO  o.a.c.f.s.ConnectionStateManager - State change: CONNECTED
 Stellar, Go!
 Please note that functions are loading lazily in the background and will be unavailable until loaded fully.
@@ -1640,7 +1247,7 @@ Returns: A Map associated with the indicator and enrichment type.  Empty otherwi
 [Stellar]&gt;&gt;&gt; non_us := whois_info.home_country != 'US'
 [Stellar]&gt;&gt;&gt; is_local := IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21')
 [Stellar]&gt;&gt;&gt; is_both := whois_info.home_country != 'US' &amp;&amp; IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21')
-[Stellar]&gt;&gt;&gt; rules := [ { 'name' : 'is non-us', 'rule' : SHELL_GET_EXPRESSION('non_us'), 'score' : 10 } , { 'name' : 'is local', 'rule' : SHELL_GET_EXPRESSION('is_local '), 'score' : 20 } , { 'name' : 'both non-us and local', 'comment' : 'union of both rules.',  'rule' : SHELL_GET_EXPRESSION('is_both'), 'score' : 50 } ]  
+[Stellar]&gt;&gt;&gt; rules := [ { 'name' : 'is non-us', 'rule' : SHELL_GET_EXPRESSION('non_us'), 'score' : 10 } , { 'name' : 'is local', 'rule' : SHELL_GET_EXPRESSION('is_local'), 'score' : 20 } , { 'name' : 'both non-us and local', 'comment' : 'union of both rules.',  'rule' : SHELL_GET_EXPRESSION('is_both'), 'score' : 50 } ]
 [Stellar]&gt;&gt;&gt; # Now that we have our rules staged, we can add them to our config.
 [Stellar]&gt;&gt;&gt; squid_enrichment_config_new := THREAT_TRIAGE_ADD( squid_enrichment_config_new, rules )
 [Stellar]&gt;&gt;&gt; THREAT_TRIAGE_PRINT(squid_enrichment_config_new)
@@ -1747,46 +1354,49 @@ SION('is_both') ] )
   &quot;configuration&quot; : { }
 }
 [Stellar]&gt;&gt;&gt; 
-</pre></div></div></div>
+</pre></div></div>
+</div>
 <div class="section">
 <h3><a name="Simulate_Threat_Triage_Rules"></a>Simulate Threat Triage Rules</h3>
-
 <ol style="list-style-type: decimal">
-  
+
 <li>
+
 <p>Create a threat triage engine.</p>
-  
-<div class="source">
-<div class="source">
-<pre>[Stellar]&gt;&gt;&gt; t := THREAT_TRIAGE_INIT()
+
+<div>
+<div>
+<pre class="source">[Stellar]&gt;&gt;&gt; t := THREAT_TRIAGE_INIT()
 [Stellar]&gt;&gt;&gt; t
 ThreatTriage{0 rule(s)}
-</pre></div></div></li>
-  
+</pre></div></div>
+</li>
 <li>
+
 <p>Add a few triage rules.</p>
-  
-<div class="source">
-<div class="source">
-<pre>[Stellar]&gt;&gt;&gt; THREAT_TRIAGE_ADD(t, {&quot;name&quot;:&quot;rule1&quot;, &quot;rule&quot;:&quot;value&gt;10&quot;, 
+
+<div>
+<div>
+<pre class="source">[Stellar]&gt;&gt;&gt; THREAT_TRIAGE_ADD(t, {&quot;name&quot;:&quot;rule1&quot;, &quot;rule&quot;:&quot;value&gt;10&quot;, &quot;score&quot;:10})
 </pre></div></div>
-  
-<div class="source">
-<div class="source">
-<pre>[Stellar]&gt;&gt;&gt; THREAT_TRIAGE_ADD(t, {&quot;name&quot;:&quot;rule2&quot;, &quot;rule&quot;:&quot;value&gt;20&quot;, &quot;score&quot;:20})
+
+<div>
+<div>
+<pre class="source">[Stellar]&gt;&gt;&gt; THREAT_TRIAGE_ADD(t, {&quot;name&quot;:&quot;rule2&quot;, &quot;rule&quot;:&quot;value&gt;20&quot;, &quot;score&quot;:20})
+</pre></div></div>
+
+<div>
+<div>
+<pre class="source">[Stellar]&gt;&gt;&gt; THREAT_TRIAGE_ADD(t, {&quot;name&quot;:&quot;rule3&quot;, &quot;rule&quot;:&quot;value&gt;30&quot;, &quot;score&quot;:30})
 </pre></div></div>
-  
-<div class="source">
-<div class="source">
-<pre>[Stellar]&gt;&gt;&gt; THREAT_TRIAGE_ADD(t, {&quot;name&quot;:&quot;rule3&quot;, &quot;rule&quot;:&quot;value&gt;30&quot;, &quot;score&quot;:30})
-</pre></div></div></li>
-  
+</li>
 <li>
+
 <p>Review the rules that you have created.</p>
-  
-<div class="source">
-<div class="source">
-<pre>[Stellar]&gt;&gt;&gt; THREAT_TRIAGE_PRINT(t)
+
+<div>
+<div>
+<pre class="source">[Stellar]&gt;&gt;&gt; THREAT_TRIAGE_PRINT(t)
 &#x2554;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2564;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2564;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2564;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2564;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2557;
 &#x2551; Name  &#x2502; Comment &#x2502; Triage Rule &#x2502; Score &#x2502; Reason &#x2551;
 &#x2560;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x256a;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x256a;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x256a;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x256a;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2563;
@@ -1796,46 +1406,49 @@ ThreatTriage{0 rule(s)}
 &#x255f;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x253c;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x253c;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x253c;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x253c;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2500;&#x2562;
 &#x2551; rule3 &#x2502;         &#x2502; value&gt;30    &#x2502; 30    &#x2502;        &#x2551;
 &#x255a;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2567;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2567;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2567;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2567;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x2550;&#x255d;
-</pre></div></div></li>
-  
+</pre></div></div>
+</li>
 <li>
+
 <p>Create a few test messages to simulate your telemetry.</p>
-  
-<div class="source">
-<div class="source">
-<pre>[Stellar]&gt;&gt;&gt; msg1 := &quot;{ \&quot;value\&quot;:22 }&quot;
+
+<div>
+<div>
+<pre class="source">[Stellar]&gt;&gt;&gt; msg1 := &quot;{ \&quot;value\&quot;:22 }&quot;
 [Stellar]&gt;&gt;&gt; msg1
 { &quot;value&quot;:22 }
 </pre></div></div>
-  
-<div class="source">
-<div class="source">
-<pre>[Stellar]&gt;&gt;&gt; msg2 := &quot;{ \&quot;value\&quot;:44 }&quot;
+
+<div>
+<div>
+<pre class="source">[Stellar]&gt;&gt;&gt; msg2 := &quot;{ \&quot;value\&quot;:44 }&quot;
 [Stellar]&gt;&gt;&gt; msg2
 { &quot;value&quot;:44 }
-</pre></div></div></li>
-  
+</pre></div></div>
+</li>
 <li>
-<p>Score a message based on the rules that have been defined. The result allows you to see the total score, the aggregator, along with details about each rule that fired.</p>
-  
-<div class="source">
-<div class="source">
-<pre>[Stellar]&gt;&gt;&gt; THREAT_TRIAGE_SCORE( msg1, t)
+
+<p>Score a message based on the rules that have been defined.  The result allows you to see the total score, the aggregator, along with details about each rule that fired.</p>
+
+<div>
+<div>
+<pre class="source">[Stellar]&gt;&gt;&gt; THREAT_TRIAGE_SCORE( msg1, t)
 {score=20.0, aggregator=MAX, rules=[{score=10.0, name=rule1, rule=value&gt;10}, {score=20.0, name=rule2, rule=value&gt;20}]}
 </pre></div></div>
-  
-<div class="source">
-<div class="source">
-<pre>[Stellar]&gt;&gt;&gt; THREAT_TRIAGE_SCORE( msg2, t)
+
+<div>
+<div>
+<pre class="source">[Stellar]&gt;&gt;&gt; THREAT_TRIAGE_SCORE( msg2, t)
 {score=30.0, aggregator=MAX, rules=[{score=10.0, name=rule1, rule=value&gt;10}, {score=20.0, name=rule2, rule=value&gt;20}, {score=30.0, name=rule3, rule=value&gt;30}]}
-</pre></div></div></li>
-  
+</pre></div></div>
+</li>
 <li>
-<p>From here you can iterate on your rule set until it does exactly what you need it to do. Once you have a working rule set, extract the configuration and push it into your live, Metron cluster.</p>
-  
-<div class="source">
-<div class="source">
-<pre>[Stellar]&gt;&gt;&gt; conf := THREAT_TRIAGE_CONFIG( t)
+
+<p>From here you can iterate on your rule set until it does exactly what you need it to do.  Once you have a working rule set, extract the configuration and push it into your live, Metron cluster.</p>
+
+<div>
+<div>
+<pre class="source">[Stellar]&gt;&gt;&gt; conf := THREAT_TRIAGE_CONFIG( t)
 [Stellar]&gt;&gt;&gt; conf
 {
   &quot;enrichment&quot; : {
@@ -1868,29 +1481,24 @@ ThreatTriage{0 rule(s)}
   &quot;configuration&quot; : { }
 }
 </pre></div></div>
-  
-<div class="source">
-<div class="source">
-<pre>[Stellar]&gt;&gt;&gt; CONFIG_PUT(&quot;ENRICHMENT&quot;, conf, &quot;bro&quot;)
-</pre></div></div></li>
-</ol></div></div>
-                  </div>
-            </div>
-          </div>
 
+<div>
+<div>
+<pre class="source">[Stellar]&gt;&gt;&gt; CONFIG_PUT(&quot;ENRICHMENT&quot;, conf, &quot;bro&quot;)
+</pre></div></div>
+</li>
+</ol></div></div>
+        </div>
+      </div>
+    </div>
     <hr/>
-
     <footer>
-            <div class="container-fluid">
-              <div class="row span12">Copyright &copy;                    2018
-                        <a href="https://www.apache.org">The Apache Software Foundation</a>.
-            All Rights Reserved.      
-                    
+      <div class="container-fluid">
+        <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+            and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+        </div>
       </div>
-
-                          
-        
-                </div>
     </footer>
   </body>
 </html>

http://git-wip-us.apache.org/repos/asf/metron/blob/8b8505da/current-book/metron-platform/metron-parsers/3rdPartyParser.html
----------------------------------------------------------------------
diff --git a/current-book/metron-platform/metron-parsers/3rdPartyParser.html b/current-book/metron-platform/metron-parsers/3rdPartyParser.html
new file mode 100644
index 0000000..988580b
--- /dev/null
+++ b/current-book/metron-platform/metron-parsers/3rdPartyParser.html
@@ -0,0 +1,467 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia Site Renderer 1.8 from src/site/markdown/metron-platform/metron-parsers/3rdPartyParser.md at 2018-06-07
+ | Rendered using Apache Maven Fluido Skin 1.7
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta name="Date-Revision-yyyymmdd" content="20180607" />
+    <meta http-equiv="Content-Language" content="en" />
+    <title>Metron &#x2013; Custom Metron Parsers</title>
+    <link rel="stylesheet" href="../../css/apache-maven-fluido-1.7.min.css" />
+    <link rel="stylesheet" href="../../css/site.css" />
+    <link rel="stylesheet" href="../../css/print.css" media="print" />
+    <script type="text/javascript" src="../../js/apache-maven-fluido-1.7.min.js"></script>
+<script type="text/javascript">
+              $( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );
+            </script>
+  </head>
+  <body class="topBarDisabled">
+    <div class="container-fluid">
+      <div id="banner">
+        <div class="pull-left"><a href="http://metron.apache.org/" id="bannerLeft"><img src="../../images/metron-logo.png"  alt="Apache Metron" width="148px" height="48px"/></a></div>
+        <div class="pull-right"></div>
+        <div class="clear"><hr/></div>
+      </div>
+
+      <div id="breadcrumbs">
+        <ul class="breadcrumb">
+      <li class=""><a href="http://www.apache.org" class="externalLink" title="Apache">Apache</a><span class="divider">/</span></li>
+      <li class=""><a href="http://metron.apache.org/" class="externalLink" title="Metron">Metron</a><span class="divider">/</span></li>
+      <li class=""><a href="../../index.html" title="Documentation">Documentation</a><span class="divider">/</span></li>
+    <li class="active ">Custom Metron Parsers</li>
+        <li id="publishDate" class="pull-right"><span class="divider">|</span> Last Published: 2018-06-07</li>
+          <li id="projectVersion" class="pull-right">Version: 0.5.0</li>
+        </ul>
+      </div>
+      <div class="row-fluid">
+        <div id="leftColumn" class="span2">
+          <div class="well sidebar-nav">
+    <ul class="nav nav-list">
+      <li class="nav-header">User Documentation</li>
+    <li><a href="../../index.html" title="Metron"><span class="icon-chevron-down"></span>Metron</a>
+    <ul class="nav nav-list">
+    <li><a href="../../CONTRIBUTING.html" title="CONTRIBUTING"><span class="none"></span>CONTRIBUTING</a></li>
+    <li><a href="../../Upgrading.html" title="Upgrading"><span class="none"></span>Upgrading</a></li>
+    <li><a href="../../metron-analytics/index.html" title="Analytics"><span class="icon-chevron-right"></span>Analytics</a></li>
+    <li><a href="../../metron-contrib/metron-docker/index.html" title="Docker"><span class="none"></span>Docker</a></li>
+    <li><a href="../../metron-contrib/metron-performance/index.html" title="Performance"><span class="none"></span>Performance</a></li>
+    <li><a href="../../metron-deployment/index.html" title="Deployment"><span class="icon-chevron-right"></span>Deployment</a></li>
+    <li><a href="../../metron-interface/metron-alerts/index.html" title="Alerts"><span class="none"></span>Alerts</a></li>
+    <li><a href="../../metron-interface/metron-config/index.html" title="Config"><span class="none"></span>Config</a></li>
+    <li><a href="../../metron-interface/metron-rest/index.html" title="Rest"><span class="none"></span>Rest</a></li>
+    <li><a href="../../metron-platform/index.html" title="Platform"><span class="icon-chevron-down"></span>Platform</a>
+    <ul class="nav nav-list">
+    <li><a href="../../metron-platform/Performance-tuning-guide.html" title="Performance-tuning-guide"><span class="none"></span>Performance-tuning-guide</a></li>
+    <li><a href="../../metron-platform/metron-api/index.html" title="Api"><span class="none"></span>Api</a></li>
+    <li><a href="../../metron-platform/metron-common/index.html" title="Common"><span class="none"></span>Common</a></li>
+    <li><a href="../../metron-platform/metron-data-management/index.html" title="Data-management"><span class="none"></span>Data-management</a></li>
+    <li><a href="../../metron-platform/metron-elasticsearch/index.html" title="Elasticsearch"><span class="none"></span>Elasticsearch</a></li>
+    <li><a href="../../metron-platform/metron-enrichment/index.html" title="Enrichment"><span class="icon-chevron-right"></span>Enrichment</a></li>
+    <li><a href="../../metron-platform/metron-indexing/index.html" title="Indexing"><span class="none"></span>Indexing</a></li>
+    <li><a href="../../metron-platform/metron-management/index.html" title="Management"><span class="none"></span>Management</a></li>
+    <li><a href="../../metron-platform/metron-parsers/index.html" title="Parsers"><span class="icon-chevron-down"></span>Parsers</a>
+    <ul class="nav nav-list">
+    <li class="active"><a href="#"><span class="none"></span>3rdPartyParser</a></li>
+    <li><a href="../../metron-platform/metron-parsers/parser-testing.html" title="parser-testing"><span class="none"></span>parser-testing</a></li>
+    </ul>
+</li>
+    <li><a href="../../metron-platform/metron-pcap-backend/index.html" title="Pcap-backend"><span class="none"></span>Pcap-backend</a></li>
+    <li><a href="../../metron-platform/metron-writer/index.html" title="Writer"><span class="none"></span>Writer</a></li>
+    </ul>
+</li>
+    <li><a href="../../metron-sensors/index.html" title="Sensors"><span class="icon-chevron-right"></span>Sensors</a></li>
+    <li><a href="../../metron-stellar/stellar-3rd-party-example/index.html" title="Stellar-3rd-party-example"><span class="none"></span>Stellar-3rd-party-example</a></li>
+    <li><a href="../../metron-stellar/stellar-common/index.html" title="Stellar-common"><span class="icon-chevron-right"></span>Stellar-common</a></li>
+    <li><a href="../../metron-stellar/stellar-zeppelin/index.html" title="Stellar-zeppelin"><span class="none"></span>Stellar-zeppelin</a></li>
+    <li><a href="../../use-cases/index.html" title="Use-cases"><span class="icon-chevron-right"></span>Use-cases</a></li>
+    </ul>
+</li>
+</ul>
+          <hr />
+          <div id="poweredBy">
+            <div class="clear"></div>
+            <div class="clear"></div>
+            <div class="clear"></div>
+            <div class="clear"></div>
+<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" /></a>
+            </div>
+          </div>
+        </div>
+        <div id="bodyColumn"  class="span10" >
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<h1>Custom Metron Parsers</h1>
+<p><a name="Custom_Metron_Parsers"></a></p>
+<p>We have many stock parsers for normal operations.  Some of these are networking and cybersecurity focused (e.g. the ASA Parser), some of these are general purpose (e.g. the CSVParser), but inevitably users will want to extend the system to process their own data formats.  To enable this, this is a walkthrough of how to create and use a custom parser within Metron.</p>
+<p><a name="Writing_A_Custom_Parser"></a></p>
+<h1>Writing A Custom Parser</h1>
+<p>Before we can use a parser, we will need to create a custom parser.  The parser is the workhorse of Metron ingest.  It provides the mapping between the raw data coming in via the Kafka value and a <tt>JSONObject</tt>, the internal data structure provided.</p>
+<div class="section">
+<h2><a name="Implementation"></a>Implementation</h2>
+<p>In order to do create a custom parser, we need to do one of the following:</p>
+<ul>
+
+<li>Write a class which conforms to the <tt>org.apache.metron.parsers.interfaces.MessageParser&lt;JSONObject&gt;</tt> and <tt>java.util.Serializable</tt> interfaces
+<ul>
+
+<li>Implement <tt>init()</tt>, <tt>validate(JSONObject message)</tt>, and <tt>List&lt;JSONObject&gt; parse(byte[] rawMessage)</tt></li>
+</ul>
+</li>
+<li>Write a class which extends <tt>org.apache.metron.parsers.BasicParser</tt>
+<ul>
+
+<li>Provides convenience implementations to <tt>validate</tt> which ensures <tt>timestamp</tt> and <tt>original_string</tt> fields exist.</li>
+</ul>
+</li>
+</ul></div>
+<div class="section">
+<h2><a name="Example"></a>Example</h2>
+<p>In order to illustrate how this might be done, let&#x2019;s create a very simple parser that takes a comma separated pair and creates a couple of fields:</p>
+<ul>
+
+<li><tt>original_string</tt> &#x2013; the raw data</li>
+<li><tt>timestamp</tt> &#x2013; the current time</li>
+<li><tt>first</tt> &#x2013; the first field of the comma separated pair</li>
+<li><tt>last</tt> &#x2013; the last field of the comma separated pair</li>
+</ul>
+<p>For this demonstration, let&#x2019;s create a maven project to compile our project.  We&#x2019;ll call it <tt>extra_parsers</tt>, so in your workspace, let&#x2019;s set up the maven project:</p>
+<ul>
+
+<li>Create the maven infrastructure for <tt>extra_parsers</tt> via</li>
+</ul>
+
+<div>
+<div>
+<pre class="source">mkdir -p extra_parsers/src/{main,test}/java
+</pre></div></div>
+
+<ul>
+
+<li>Create a pom file indicating how we should build our parsers by editing <tt>extra_parsers/pom.xml</tt> with the following content:</li>
+</ul>
+
+<div>
+<div>
+<pre class="source">&lt;project xmlns=&quot;http://maven.apache.org/POM/4.0.0&quot; xmlns:xsi=&quot;http://www.w3.org/2001/XMLSchema-instance&quot;
+  xsi:schemaLocation=&quot;http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd&quot;&gt;
+  &lt;modelVersion&gt;4.0.0&lt;/modelVersion&gt;
+  &lt;groupId&gt;com.3rdparty&lt;/groupId&gt;
+  &lt;artifactId&gt;extra-parsers&lt;/artifactId&gt;
+  &lt;packaging&gt;jar&lt;/packaging&gt;
+  &lt;version&gt;1.0-SNAPSHOT&lt;/version&gt;
+  &lt;name&gt;extra-parsers&lt;/name&gt;
+  &lt;url&gt;http://thirdpartysoftware.org&lt;/url&gt;
+  &lt;properties&gt;
+    &lt;!-- The java version to conform to.  Metron works all the way to 1.8 --&gt;
+    &lt;java_version&gt;1.8&lt;/java_version&gt;
+    &lt;!-- The version of Metron that we'll be targetting. --&gt;
+    &lt;metron_version&gt;0.4.1&lt;/metron_version&gt;
+    &lt;!-- To complete the simulation, we'll depend on a common dependency --&gt;
+    &lt;guava_version&gt;19.0&lt;/guava_version&gt;
+    &lt;!-- We will shade our dependencies to create a single jar at the end --&gt;
+    &lt;shade_version&gt;2.4.3&lt;/shade_version&gt;
+  &lt;/properties&gt;
+  &lt;dependencies&gt;
+    &lt;!--
+    We want to depend on Metron, but ensure that the scope is &quot;provided&quot;
+    as we do not want to include it in our bundle.
+    --&gt;
+    &lt;dependency&gt;
+      &lt;groupId&gt;org.apache.metron&lt;/groupId&gt;
+      &lt;artifactId&gt;metron-parsers&lt;/artifactId&gt;
+      &lt;version&gt;${metron_version}&lt;/version&gt;
+      &lt;scope&gt;provided&lt;/scope&gt;
+    &lt;/dependency&gt;
+    &lt;dependency&gt;
+      &lt;groupId&gt;com.google.guava&lt;/groupId&gt;
+      &lt;artifactId&gt;guava&lt;/artifactId&gt;
+      &lt;version&gt;${guava_version}&lt;/version&gt;
+    &lt;/dependency&gt;
+    &lt;dependency&gt;
+      &lt;groupId&gt;junit&lt;/groupId&gt;
+      &lt;artifactId&gt;junit&lt;/artifactId&gt;
+      &lt;version&gt;3.8.1&lt;/version&gt;
+      &lt;scope&gt;test&lt;/scope&gt;
+    &lt;/dependency&gt;
+  &lt;/dependencies&gt;
+  &lt;build&gt;
+    &lt;plugins&gt;
+     &lt;!-- We will set up the shade plugin to create a single jar at the
+           end of the build lifecycle.  We will exclude some things and
+           relocate others to simulate a real situation.
+           
+           One thing to note is that it's a good practice to shade and
+           relocate common libraries that may be dependencies in Metron.
+           Your jar will be merged with the parsers jar, so the metron
+           version will be included for all overlapping classes.
+           So, shade and relocate to ensure that YOUR version of the library is used.
+      --&gt;
+
+      &lt;plugin&gt;
+        &lt;groupId&gt;org.apache.maven.plugins&lt;/groupId&gt;
+        &lt;artifactId&gt;maven-shade-plugin&lt;/artifactId&gt;
+        &lt;version&gt;${shade_version}&lt;/version&gt;
+        &lt;configuration&gt;
+          &lt;createDependencyReducedPom&gt;true&lt;/createDependencyReducedPom&gt;
+          &lt;artifactSet&gt;
+            &lt;excludes&gt;
+              &lt;!-- Exclude slf4j for no reason other than to illustrate how to exclude dependencies.
+                   The metron team has nothing against slf4j. :-)
+               --&gt;
+              &lt;exclude&gt;*slf4j*&lt;/exclude&gt;
+            &lt;/excludes&gt;
+          &lt;/artifactSet&gt;
+        &lt;/configuration&gt;
+        &lt;executions&gt;
+          &lt;execution&gt;
+            &lt;phase&gt;package&lt;/phase&gt;
+            &lt;goals&gt;
+              &lt;goal&gt;shade&lt;/goal&gt;
+            &lt;/goals&gt;
+            &lt;configuration&gt;
+              &lt;shadedArtifactAttached&gt;true&lt;/shadedArtifactAttached&gt;
+              &lt;shadedClassifierName&gt;uber&lt;/shadedClassifierName&gt;
+              &lt;filters&gt;
+                &lt;filter&gt;
+                  &lt;!-- Sometimes these get added and confuse the uber jar out of shade --&gt;
+                  &lt;artifact&gt;*:*&lt;/artifact&gt;
+                  &lt;excludes&gt;
+                    &lt;exclude&gt;META-INF/*.SF&lt;/exclude&gt;
+                    &lt;exclude&gt;META-INF/*.DSA&lt;/exclude&gt;
+                    &lt;exclude&gt;META-INF/*.RSA&lt;/exclude&gt;
+                  &lt;/excludes&gt;
+                &lt;/filter&gt;
+              &lt;/filters&gt;
+              &lt;relocations&gt;
+                &lt;!-- Relocate guava as it's used in Metron and I really want 0.19 --&gt;
+                &lt;relocation&gt;
+                  &lt;pattern&gt;com.google&lt;/pattern&gt;
+                  &lt;shadedPattern&gt;com.thirdparty.guava&lt;/shadedPattern&gt;
+                &lt;/relocation&gt;
+              &lt;/relocations&gt;
+              &lt;artifactSet&gt;
+                &lt;excludes&gt;
+                  &lt;!-- We can also exclude by artifactId and groupId --&gt;
+                  &lt;exclude&gt;storm:storm-core:*&lt;/exclude&gt;
+                  &lt;exclude&gt;storm:storm-lib:*&lt;/exclude&gt;
+                  &lt;exclude&gt;org.slf4j.impl*&lt;/exclude&gt;
+                  &lt;exclude&gt;org.slf4j:slf4j-log4j*&lt;/exclude&gt;
+                &lt;/excludes&gt;
+              &lt;/artifactSet&gt;
+            &lt;/configuration&gt;
+          &lt;/execution&gt;
+        &lt;/executions&gt;
+      &lt;/plugin&gt;
+      &lt;!--
+      We want to make sure we compile using java 1.8.
+      --&gt;
+      &lt;plugin&gt;
+        &lt;groupId&gt;org.apache.maven.plugins&lt;/groupId&gt;
+        &lt;artifactId&gt;maven-compiler-plugin&lt;/artifactId&gt;
+        &lt;version&gt;3.5.1&lt;/version&gt;
+        &lt;configuration&gt;
+          &lt;forceJavacCompilerUse&gt;true&lt;/forceJavacCompilerUse&gt;
+          &lt;source&gt;${java_version}&lt;/source&gt;
+          &lt;compilerArgument&gt;-Xlint:unchecked&lt;/compilerArgument&gt;
+          &lt;target&gt;${java_version}&lt;/target&gt;
+          &lt;showWarnings&gt;true&lt;/showWarnings&gt;
+        &lt;/configuration&gt;
+      &lt;/plugin&gt;
+    &lt;/plugins&gt;
+  &lt;/build&gt;
+&lt;/project&gt;
+</pre></div></div>
+
+<ul>
+
+<li>Now let&#x2019;s create our parser  <tt>com.thirdparty.SimpleParser</tt> by creating the file <tt>extra-parsers/src/main/java/com/thirdparty/SimpleParser.java</tt> with the following content:</li>
+</ul>
+
+<div>
+<div>
+<pre class="source">package com.thirdparty;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+
+import java.util.List;
+import java.util.Map;
+
+public class SimpleParser extends BasicParser {
+  @Override
+  public void init() {
+
+  }
+
+  @Override
+  public List&lt;JSONObject&gt; parse(byte[] bytes) {
+    String input = new String(bytes);
+    Iterable&lt;String&gt; it = Splitter.on(&quot;,&quot;).split(input);
+    JSONObject ret = new JSONObject();
+    ret.put(&quot;original_string&quot;, input);
+    ret.put(&quot;timestamp&quot;, System.currentTimeMillis());
+    ret.put(&quot;first&quot;, Iterables.getFirst(it, &quot;missing&quot;));
+    ret.put(&quot;last&quot;, Iterables.getLast(it, &quot;missing&quot;));
+    return ImmutableList.of(ret);
+  }
+
+  @Override
+  public void configure(Map&lt;String, Object&gt; map) {
+
+  }
+}
+</pre></div></div>
+
+<ul>
+
+<li>Compile the parser via <tt>mvn clean package</tt> in <tt>extra_parsers</tt></li>
+</ul>
+<p>This will create a jar containing your parser and its dependencies (sans Metron dependencies) in <tt>extra-parsers/target/extra-parsers-1.0-SNAPSHOT-uber.jar</tt></p>
+<p><a name="Deploying_Your_Custom_Parser"></a></p>
+<h1>Deploying Your Custom Parser</h1>
+<p>In order to deploy your newly built custom parser, you would place the jar file above in the <tt>$METRON_HOME/parser_contrib</tt> directory on the Metron host (i.e. any host you would start parsers from or, alternatively, where the Metron REST is hosted).</p></div>
+<div class="section">
+<h2><a name="Example"></a>Example</h2>
+<p>Let&#x2019;s work through deploying the example above.</p>
+<div class="section">
+<h3><a name="Preliminaries"></a>Preliminaries</h3>
+<p>We assume that the following environment variables are set:</p>
+<ul>
+
+<li><tt>METRON_HOME</tt> - the home directory for metron</li>
+<li><tt>ZOOKEEPER</tt> - The zookeeper quorum (comma separated with port specified: e.g. <tt>node1:2181</tt> for full-dev)</li>
+<li><tt>BROKERLIST</tt> - The Kafka broker list (comma separated with port specified: e.g. <tt>node1:6667</tt> for full-dev)</li>
+<li><tt>ES_HOST</tt> - The elasticsearch master (and port) e.g. <tt>node1:9200</tt> for full-dev.</li>
+</ul>
+<p>Also, this does not assume that you are using a kerberized cluster.  If you are, then the parser start command will adjust slightly to include the security protocol.</p></div>
+<div class="section">
+<h3><a name="Copy_the_jar_file_up"></a>Copy the jar file up</h3>
+<p>Copy the jar file located in <tt>extra-parsers/target/extra-parsers-1.0-SNAPSHOT-uber.jar</tt> to <tt>$METRON_HOME/parser_contrib</tt> and ensure the permissions are such that the <tt>metron</tt> user can read and execute.</p></div>
+<div class="section">
+<h3><a name="Restart_the_REST_service_in_Ambari"></a>Restart the REST service in Ambari</h3>
+<p>In order for new parsers to be picked up, the REST service must be restarted.  You can do that from within Ambari by restarting the <tt>Metron REST</tt> service.</p></div>
+<div class="section">
+<h3><a name="Push_the_Zookeeper_Configs"></a>Push the Zookeeper Configs</h3>
+<p>Now push the config to Zookeeper with the following command: <tt>$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i $METRON_HOME/config/zookeeper/ -z $ZOOKEEPER</tt></p></div>
+<div class="section">
+<h3><a name="Create_a_Kafka_Topic"></a>Create a Kafka Topic</h3>
+<p>Create a kafka topic, let&#x2019;s call it <tt>test</tt> via: <tt>/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --zookeeper $ZOOKEEPER --create --topic test --partitions 1 --replication-factor 1</tt></p>
+<p>Note, in a real deployment, that topic would be named something more descriptive and would have replication factor and partitions set to something less trivial.</p></div>
+<div class="section">
+<h3><a name="Configure_Test_Parser"></a>Configure Test Parser</h3>
+<p>Create the a file called <tt>$METRON_HOME/config/zookeeper/parsers/test.json</tt> with the following content:</p>
+
+<div>
+<div>
+<pre class="source">{
+  &quot;parserClassName&quot;:&quot;com.thirdparty.SimpleParser&quot;,
+  &quot;sensorTopic&quot;:&quot;test&quot;
+}
+</pre></div></div>
+</div>
+<div class="section">
+<h3><a name="Start_Parser"></a>Start Parser</h3>
+<p>Now we can start the parser and send some data through:</p>
+<ul>
+
+<li>Start the parser</li>
+</ul>
+
+<div>
+<div>
+<pre class="source">$METRON_HOME/bin/start_parser_topology.sh -k $BROKERLIST -z $ZOOKEEPER -s test
+</pre></div></div>
+
+<ul>
+
+<li>Send example data through:</li>
+</ul>
+
+<div>
+<div>
+<pre class="source">echo &quot;apache,metron&quot; | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $BROKERLIST --topic test
+</pre></div></div>
+
+<ul>
+
+<li>Validate data was written in ES:</li>
+</ul>
+
+<div>
+<div>
+<pre class="source">curl -XPOST &quot;http://$ES_HOST/test*/_search?pretty&quot; -d '
+{
+  &quot;_source&quot; : [ &quot;original_string&quot;, &quot;timestamp&quot;, &quot;first&quot;, &quot;last&quot;]
+}
+'
+</pre></div></div>
+
+<p>This should yield something like:</p>
+
+<div>
+<div>
+<pre class="source">{
+  &quot;took&quot; : 23,
+  &quot;timed_out&quot; : false,
+  &quot;_shards&quot; : {
+    &quot;total&quot; : 1,
+    &quot;successful&quot; : 1,
+    &quot;failed&quot; : 0
+  },
+  &quot;hits&quot; : {
+    &quot;total&quot; : 1,
+    &quot;max_score&quot; : 1.0,
+    &quot;hits&quot; : [ {
+      &quot;_index&quot; : &quot;test_index_2017.10.04.17&quot;,
+      &quot;_type&quot; : &quot;test_doc&quot;,
+      &quot;_id&quot; : &quot;3ae4dd4d-8c09-4f2a-93c0-26ec5508baaa&quot;,
+      &quot;_score&quot; : 1.0,
+      &quot;_source&quot; : {
+        &quot;original_string&quot; : &quot;apache,metron&quot;,
+        &quot;last&quot; : &quot;metron&quot;,
+        &quot;first&quot; : &quot;apache&quot;,
+        &quot;timestamp&quot; : 1507138373223
+      }
+    } ]
+  }
+}
+</pre></div></div>
+</div>
+<div class="section">
+<h3><a name="Via_the_Management_UI"></a>Via the Management UI</h3>
+<p>As long as the REST service is restarted after new parsers are added to <tt>$METRON_HOME/parser_contrib</tt>, they are available in the UI to creating and deploying parsers.</p></div></div>
+        </div>
+      </div>
+    </div>
+    <hr/>
+    <footer>
+      <div class="container-fluid">
+        <div class="row-fluid">
+© 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache, the Apache feather logo,
+            and the Apache Metron project logo are trademarks of The Apache Software Foundation.
+        </div>
+      </div>
+    </footer>
+  </body>
+</html>


Mime
View raw message