metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject svn commit: r19216 [8/15] - in /dev/incubator/metron/0.4.0-RC1-incubating/book-site: ./ css/ images/ images/logos/ images/profiles/ img/ js/ metron-analytics/ metron-analytics/metron-maas-service/ metron-analytics/metron-profiler-client/ metron-analyti...
Date Thu, 20 Apr 2017 05:38:46 GMT
Added: dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-deployment/vagrant/fastcapa-test-platform/index.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-deployment/vagrant/fastcapa-test-platform/index.html (added)
+++ dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-deployment/vagrant/fastcapa-test-platform/index.html Thu Apr 20 05:38:46 2017
@@ -0,0 +1,334 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-19
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta name="Date-Revision-yyyymmdd" content="20170419" />
+    <meta http-equiv="Content-Language" content="en" />
+    <title>Metron &#x2013; Fastcapa Test Environment</title>
+    <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.3.0.min.css" />
+    <link rel="stylesheet" href="../../../css/site.css" />
+    <link rel="stylesheet" href="../../../css/print.css" media="print" />
+
+      
+    <script type="text/javascript" src="../../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+                          
+        
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+          
+            </head>
+        <body class="topBarDisabled">
+          
+                
+                    
+    
+        <div class="container-fluid">
+          <div id="banner">
+        <div class="pull-left">
+                                    <a href="http://metron.incubator.apache.org/" id="bannerLeft">
+                                                                                                <img src="../../../images/metron-logo.png"  alt="Apache Metron - Incubating" width="148px" height="48px"/>
+                </a>
+                      </div>
+        <div class="pull-right">                  <a href="http://incubator.apache.org/" id="bannerRight">
+                                                                                                <img src="../../../images/ApacheIncubating_Logo.png"  alt="Apache Incubating" width="192px" height="48px"/>
+                </a>
+      </div>
+        <div class="clear"><hr/></div>
+      </div>
+
+      <div id="breadcrumbs">
+        <ul class="breadcrumb">
+                
+                    
+                              <li class="">
+                    <a href="http://www.apache.org" class="externalLink" title="Apache">
+        Apache</a>
+        </li>
+      <li class="divider ">/</li>
+            <li class="">
+                    <a href="http://metron.incubator.apache.org/" class="externalLink" title="Metron-Incubating">
+        Metron-Incubating</a>
+        </li>
+      <li class="divider ">/</li>
+            <li class="">
+                    <a href="../../../index.html" title="Documentation">
+        Documentation</a>
+        </li>
+      <li class="divider ">/</li>
+        <li class="">Fastcapa Test Environment</li>
+        
+                
+                    
+                  <li id="publishDate" class="pull-right">Last Published: 2017-04-19</li> <li class="divider pull-right">|</li>
+              <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+            
+                            </ul>
+      </div>
+
+            
+      <div class="row-fluid">
+        <div id="leftColumn" class="span3">
+          <div class="well sidebar-nav">
+                
+                    
+                <ul class="nav nav-list">
+                    <li class="nav-header">User Documentation</li>
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+      <li>
+    
+                          <a href="../../../index.html" title="Metron">
+          <i class="icon-chevron-down"></i>
+        Metron</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../../Upgrading.html" title="Upgrading">
+          <i class="none"></i>
+        Upgrading</a>
+            </li>
+                                                                                                                                                      
+      <li>
+    
+                          <a href="../../../metron-analytics/index.html" title="Analytics">
+          <i class="icon-chevron-right"></i>
+        Analytics</a>
+                  </li>
+                                                                                                                                                                                                                                                                                                                                                                            
+      <li>
+    
+                          <a href="../../../metron-deployment/index.html" title="Deployment">
+          <i class="icon-chevron-down"></i>
+        Deployment</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2">
+          <i class="none"></i>
+        Amazon-ec2</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker">
+          <i class="none"></i>
+        Ansible-docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker">
+          <i class="none"></i>
+        Rpm-docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build">
+          <i class="none"></i>
+        Packer-build</a>
+            </li>
+                                                                                                                                                
+      <li>
+    
+                          <a href="../../../metron-deployment/roles/index.html" title="Roles">
+          <i class="icon-chevron-right"></i>
+        Roles</a>
+                  </li>
+                                                                                                                                                          
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/index.html" title="Vagrant">
+          <i class="icon-chevron-down"></i>
+        Vagrant</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/Kerberos-setup.html" title="Kerberos-setup">
+          <i class="none"></i>
+        Kerberos-setup</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/codelab-platform/index.html" title="Codelab-platform">
+          <i class="none"></i>
+        Codelab-platform</a>
+            </li>
+                      
+      <li class="active">
+    
+            <a href="#"><i class="none"></i>Fastcapa-test-platform</a>
+          </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/full-dev-platform/index.html" title="Full-dev-platform">
+          <i class="none"></i>
+        Full-dev-platform</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/quick-dev-platform/index.html" title="Quick-dev-platform">
+          <i class="none"></i>
+        Quick-dev-platform</a>
+            </li>
+              </ul>
+        </li>
+              </ul>
+        </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-docker/index.html" title="Docker">
+          <i class="none"></i>
+        Docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-interface/metron-config/index.html" title="Config">
+          <i class="none"></i>
+        Config</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-interface/metron-rest/index.html" title="Rest">
+          <i class="none"></i>
+        Rest</a>
+            </li>
+                                                                                                                                                                                                                                                
+      <li>
+    
+                          <a href="../../../metron-platform/index.html" title="Platform">
+          <i class="icon-chevron-right"></i>
+        Platform</a>
+                  </li>
+                                                                                                            
+      <li>
+    
+                          <a href="../../../metron-sensors/index.html" title="Sensors">
+          <i class="icon-chevron-right"></i>
+        Sensors</a>
+                  </li>
+              </ul>
+        </li>
+            </ul>
+                
+                    
+                
+          <hr class="divider" />
+
+           <div id="poweredBy">
+                            <div class="clear"></div>
+                            <div class="clear"></div>
+                            <div class="clear"></div>
+                             <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+        <img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" />
+      </a>
+                  </div>
+          </div>
+        </div>
+        
+                
+        <div id="bodyColumn"  class="span9" >
+                                  
+            <h1>Fastcapa Test Environment</h1>
+<p>Provides a test environment for the development and testing of Fastcapa. The environment is automatically validated after it is created to ensure that Fastcapa is behaving correctly.</p>
+<p>Two virtualized nodes are launched with Vagrant that can communicate with one another over a private network. </p>
+
+<ul>
+  
+<li>The <tt>source</tt> node uses Metron&#x2019;s <tt>pcap_replay</tt> functionality to transmit raw network packet data over a private network.</li>
+  
+<li>The <tt>sink</tt> node is running <tt>fastcapa</tt> and is capturing these network packets.</li>
+  
+<li>Fastcapa then transforms and bundles the packets into a message.</li>
+  
+<li>The message is sent to a Kafka broker running on the <tt>source</tt> node.</li>
+</ul>
+<div class="section">
+<h2><a name="Getting_Started"></a>Getting Started</h2>
+<p>Simply run <tt>vagrant up</tt> to launch the environment. Automated tests are executed after provisioning completes to ensure that Fastcapa and the rest of the environment is functioning properly.</p>
+
+<div class="source">
+<div class="source">
+<pre>$ vagrant up
+==&gt; source: Running provisioner: ansible...
+    source: Running ansible-playbook...
+...
+TASK [debug] *******************************************************************
+ok: [source] =&gt; {
+    &quot;msg&quot;: &quot;Successfully received packets sent from pcap-replay!&quot;
+}
+...
+TASK [debug] *******************************************************************
+ok: [source] =&gt; {
+    &quot;msg&quot;: &quot;Successfully received a Kafka message from fastcapa!&quot;
+}
+</pre></div></div></div>
+<div class="section">
+<h2><a name="Going_Deeper"></a>Going Deeper</h2>
+<p>This section will outline in more detail the environment and how to interact with it.</p>
+<div class="section">
+<h3><a name="source"></a><tt>source</tt></h3>
+<p>To validate that the <tt>source</tt> node is functioning properly, run the following commands.</p>
+<p>First, ensure that the <tt>pcap-replay</tt> service is running.</p>
+
+<div class="source">
+<div class="source">
+<pre>vagrant ssh source
+sudo service pcap-replay status
+</pre></div></div>
+<p>Use <tt>tcpdump</tt> to ensure that the raw packet data is being sent over the private network. Enter &#x2018;CTRL-C&#x2019; to kill the <tt>tcpdump</tt> process once you are able to see that packets are being sent.</p>
+
+<div class="source">
+<div class="source">
+<pre>sudo yum -y install tcpdump
+sudo tcpdump -i enp0s8
+</pre></div></div></div>
+<div class="section">
+<h3><a name="sink"></a><tt>sink</tt></h3>
+<p>Next validate that the <tt>sink</tt> is functioning properly. Run the following commands starting from the host operating system. </p>
+<p>First, ensure that the <tt>fastcapa</tt> service is running.</p>
+
+<div class="source">
+<div class="source">
+<pre>vagrant ssh sink
+service fastcapa status
+</pre></div></div>
+<p>Ensure that the raw network packet data is being received by Kafka.</p>
+
+<div class="source">
+<div class="source">
+<pre>/usr/hdp/current/kafka-broker/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic pcap
+</pre></div></div>
+<p>Enter &#x2018;CTRL-C&#x2019; to kill the <tt>kafka-console-consumer</tt> process once you are able to see that packets are being sent. These packets will appear to be gibberish in the console. This is the raw binary network packet data after all.</p></div></div>
+                  </div>
+            </div>
+          </div>
+
+    <hr/>
+
+    <footer>
+            <div class="container-fluid">
+              <div class="row span12">Copyright &copy;                   2017.
+          All Rights Reserved.      
+                    
+      </div>
+
+                          
+        
+                </div>
+    </footer>
+  </body>
+</html>

Added: dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-deployment/vagrant/full-dev-platform/index.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-deployment/vagrant/full-dev-platform/index.html (added)
+++ dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-deployment/vagrant/full-dev-platform/index.html Thu Apr 20 05:38:46 2017
@@ -0,0 +1,387 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-19
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta name="Date-Revision-yyyymmdd" content="20170419" />
+    <meta http-equiv="Content-Language" content="en" />
+    <title>Metron &#x2013; Full Development Platform</title>
+    <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.3.0.min.css" />
+    <link rel="stylesheet" href="../../../css/site.css" />
+    <link rel="stylesheet" href="../../../css/print.css" media="print" />
+
+      
+    <script type="text/javascript" src="../../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+                          
+        
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+          
+            </head>
+        <body class="topBarDisabled">
+          
+                
+                    
+    
+        <div class="container-fluid">
+          <div id="banner">
+        <div class="pull-left">
+                                    <a href="http://metron.incubator.apache.org/" id="bannerLeft">
+                                                                                                <img src="../../../images/metron-logo.png"  alt="Apache Metron - Incubating" width="148px" height="48px"/>
+                </a>
+                      </div>
+        <div class="pull-right">                  <a href="http://incubator.apache.org/" id="bannerRight">
+                                                                                                <img src="../../../images/ApacheIncubating_Logo.png"  alt="Apache Incubating" width="192px" height="48px"/>
+                </a>
+      </div>
+        <div class="clear"><hr/></div>
+      </div>
+
+      <div id="breadcrumbs">
+        <ul class="breadcrumb">
+                
+                    
+                              <li class="">
+                    <a href="http://www.apache.org" class="externalLink" title="Apache">
+        Apache</a>
+        </li>
+      <li class="divider ">/</li>
+            <li class="">
+                    <a href="http://metron.incubator.apache.org/" class="externalLink" title="Metron-Incubating">
+        Metron-Incubating</a>
+        </li>
+      <li class="divider ">/</li>
+            <li class="">
+                    <a href="../../../index.html" title="Documentation">
+        Documentation</a>
+        </li>
+      <li class="divider ">/</li>
+        <li class="">Full Development Platform</li>
+        
+                
+                    
+                  <li id="publishDate" class="pull-right">Last Published: 2017-04-19</li> <li class="divider pull-right">|</li>
+              <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+            
+                            </ul>
+      </div>
+
+            
+      <div class="row-fluid">
+        <div id="leftColumn" class="span3">
+          <div class="well sidebar-nav">
+                
+                    
+                <ul class="nav nav-list">
+                    <li class="nav-header">User Documentation</li>
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+      <li>
+    
+                          <a href="../../../index.html" title="Metron">
+          <i class="icon-chevron-down"></i>
+        Metron</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../../Upgrading.html" title="Upgrading">
+          <i class="none"></i>
+        Upgrading</a>
+            </li>
+                                                                                                                                                      
+      <li>
+    
+                          <a href="../../../metron-analytics/index.html" title="Analytics">
+          <i class="icon-chevron-right"></i>
+        Analytics</a>
+                  </li>
+                                                                                                                                                                                                                                                                                                                                                                            
+      <li>
+    
+                          <a href="../../../metron-deployment/index.html" title="Deployment">
+          <i class="icon-chevron-down"></i>
+        Deployment</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2">
+          <i class="none"></i>
+        Amazon-ec2</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker">
+          <i class="none"></i>
+        Ansible-docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker">
+          <i class="none"></i>
+        Rpm-docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build">
+          <i class="none"></i>
+        Packer-build</a>
+            </li>
+                                                                                                                                                
+      <li>
+    
+                          <a href="../../../metron-deployment/roles/index.html" title="Roles">
+          <i class="icon-chevron-right"></i>
+        Roles</a>
+                  </li>
+                                                                                                                                                          
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/index.html" title="Vagrant">
+          <i class="icon-chevron-down"></i>
+        Vagrant</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/Kerberos-setup.html" title="Kerberos-setup">
+          <i class="none"></i>
+        Kerberos-setup</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/codelab-platform/index.html" title="Codelab-platform">
+          <i class="none"></i>
+        Codelab-platform</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/fastcapa-test-platform/index.html" title="Fastcapa-test-platform">
+          <i class="none"></i>
+        Fastcapa-test-platform</a>
+            </li>
+                      
+      <li class="active">
+    
+            <a href="#"><i class="none"></i>Full-dev-platform</a>
+          </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/quick-dev-platform/index.html" title="Quick-dev-platform">
+          <i class="none"></i>
+        Quick-dev-platform</a>
+            </li>
+              </ul>
+        </li>
+              </ul>
+        </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-docker/index.html" title="Docker">
+          <i class="none"></i>
+        Docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-interface/metron-config/index.html" title="Config">
+          <i class="none"></i>
+        Config</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-interface/metron-rest/index.html" title="Rest">
+          <i class="none"></i>
+        Rest</a>
+            </li>
+                                                                                                                                                                                                                                                
+      <li>
+    
+                          <a href="../../../metron-platform/index.html" title="Platform">
+          <i class="icon-chevron-right"></i>
+        Platform</a>
+                  </li>
+                                                                                                            
+      <li>
+    
+                          <a href="../../../metron-sensors/index.html" title="Sensors">
+          <i class="icon-chevron-right"></i>
+        Sensors</a>
+                  </li>
+              </ul>
+        </li>
+            </ul>
+                
+                    
+                
+          <hr class="divider" />
+
+           <div id="poweredBy">
+                            <div class="clear"></div>
+                            <div class="clear"></div>
+                            <div class="clear"></div>
+                             <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+        <img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" />
+      </a>
+                  </div>
+          </div>
+        </div>
+        
+                
+        <div id="bodyColumn"  class="span9" >
+                                  
+            <h1>Full Development Platform</h1>
+<p>This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running on Virtualbox. </p>
+<p>Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. To that end the vagrant file configuration defaults to disabling solr and yaf.</p>
+<div class="section">
+<h2><a name="Getting_Started"></a>Getting Started</h2>
+<div class="section">
+<h3><a name="Prerequisites"></a>Prerequisites</h3>
+<p>The computer used to deploy Apache Metron will need to have the following components installed.</p>
+
+<ul>
+  
+<li><a class="externalLink" href="https://github.com/ansible/ansible">Ansible</a> (2.0.0.2 or 2.2.2.0)</li>
+  
+<li><a class="externalLink" href="https://www.vagrantup.com">Vagrant</a> 1.8.1</li>
+  
+<li><a class="externalLink" href="https://virtualbox.org">Virtualbox</a> 5.0.16</li>
+  
+<li>Python 2.7.11</li>
+  
+<li>Maven 3.3.9</li>
+</ul>
+<div class="section">
+<h4><a name="macOS"></a>macOS</h4>
+<p>Any platform that supports these tools is suitable, but the following instructions cover installation on macOS. The easiest means of installing these tools on a Mac is to use the excellent <a class="externalLink" href="http://brew.sh/">Homebrew</a> project.</p>
+
+<ol style="list-style-type: decimal">
+  
+<li>
+<p>Install Homebrew by following the instructions at <a class="externalLink" href="http://brew.sh/">Homebrew</a>.</p></li>
+  
+<li>
+<p>Run the following command in a terminal to install all of the required tools.</p>
+  
+<div class="source">
+<div class="source">
+<pre>  brew cask install vagrant virtualbox java
+  brew install maven git
+</pre></div></div></li>
+  
+<li>
+<p>Install Ansible by following the instructions <a class="externalLink" href="http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip">here</a>.</p></li>
+</ol></div></div>
+<div class="section">
+<h3><a name="Deploy_Metron"></a>Deploy Metron</h3>
+
+<ol style="list-style-type: decimal">
+  
+<li>
+<p>Install Vagrant Hostmanager.</p>
+  
+<div class="source">
+<div class="source">
+<pre>  vagrant plugin install vagrant-hostmanager
+</pre></div></div></li>
+  
+<li>
+<p>Deploy Metron</p>
+  
+<div class="source">
+<div class="source">
+<pre>  cd metron-deployment/vagrant/full-dev-platform
+  vagrant up
+</pre></div></div>
+<p>Should the process fail before completing the deployment, the following command will continue the deployment process without re-instantiating the host.</p>
+  
+<div class="source">
+<div class="source">
+<pre>  vagrant provision
+</pre></div></div></li>
+</ol></div>
+<div class="section">
+<h3><a name="Explore_Metron"></a>Explore Metron</h3>
+<p>Navigate to the following resources to explore your newly minted Apache Metron environment.</p>
+
+<ul>
+  
+<li><a class="externalLink" href="http://node1:8080">Ambari</a></li>
+  
+<li><a class="externalLink" href="http://node1:5000">Metron</a></li>
+  
+<li><a class="externalLink" href="http://node1:2812">Services</a></li>
+</ul>
+<p>Connecting to the host through SSH is as simple as running the following command.</p>
+
+<div class="source">
+<div class="source">
+<pre>vagrant ssh
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Working_with_Metron"></a>Working with Metron</h3>
+<p>In addition to re-running the entire provisioning play book, you may now re-run an individual Ansible tag or a collection of tags in the following ways. The following commands will re-run the <tt>web</tt> role on the Vagrant image. This will install components (if necessary) and start the UI.</p>
+
+<div class="source">
+<div class="source">
+<pre>./run_ansible_role.sh web
+</pre></div></div>
+<p>or</p>
+
+<div class="source">
+<div class="source">
+<pre>vagrant --ansible-tags=&quot;web&quot; provision
+</pre></div></div>
+<div class="section">
+<h4><a name="Using_Tags"></a>Using Tags</h4>
+<p>A collection of tags is specified as a comma separated list.</p>
+
+<div class="source">
+<div class="source">
+<pre>./run_ansible_role.sh &quot;sensors,enrichment&quot;
+
+</pre></div></div>
+<p>Tags are listed in the playbooks, some frequently used tags:</p>
+
+<ul>
+  
+<li><tt>hdp-install</tt> - Install HDP</li>
+  
+<li><tt>hdp-deploy</tt> - Deploy and Start HDP Services (will start all Hadoop Services)</li>
+  
+<li><tt>sensors</tt> - Deploy and Start Sensors.</li>
+  
+<li><tt>enrichment</tt> - Deploy and Start Enrichment Topology.</li>
+</ul>
+<p>Note also that there is a convenience script <tt>./run_enrichment_role.sh</tt> which executes Vagrant with the <tt>enrichment</tt> tag.</p></div></div></div>
+                  </div>
+            </div>
+          </div>
+
+    <hr/>
+
+    <footer>
+            <div class="container-fluid">
+              <div class="row span12">Copyright &copy;                   2017.
+          All Rights Reserved.      
+                    
+      </div>
+
+                          
+        
+                </div>
+    </footer>
+  </body>
+</html>

Added: dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-deployment/vagrant/index.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-deployment/vagrant/index.html (added)
+++ dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-deployment/vagrant/index.html Thu Apr 20 05:38:46 2017
@@ -0,0 +1,278 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-19
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta name="Date-Revision-yyyymmdd" content="20170419" />
+    <meta http-equiv="Content-Language" content="en" />
+    <title>Metron &#x2013; Vagrant Deployment</title>
+    <link rel="stylesheet" href="../../css/apache-maven-fluido-1.3.0.min.css" />
+    <link rel="stylesheet" href="../../css/site.css" />
+    <link rel="stylesheet" href="../../css/print.css" media="print" />
+
+      
+    <script type="text/javascript" src="../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+                          
+        
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+          
+            </head>
+        <body class="topBarDisabled">
+          
+                
+                    
+    
+        <div class="container-fluid">
+          <div id="banner">
+        <div class="pull-left">
+                                    <a href="http://metron.incubator.apache.org/" id="bannerLeft">
+                                                                                                <img src="../../images/metron-logo.png"  alt="Apache Metron - Incubating" width="148px" height="48px"/>
+                </a>
+                      </div>
+        <div class="pull-right">                  <a href="http://incubator.apache.org/" id="bannerRight">
+                                                                                                <img src="../../images/ApacheIncubating_Logo.png"  alt="Apache Incubating" width="192px" height="48px"/>
+                </a>
+      </div>
+        <div class="clear"><hr/></div>
+      </div>
+
+      <div id="breadcrumbs">
+        <ul class="breadcrumb">
+                
+                    
+                              <li class="">
+                    <a href="http://www.apache.org" class="externalLink" title="Apache">
+        Apache</a>
+        </li>
+      <li class="divider ">/</li>
+            <li class="">
+                    <a href="http://metron.incubator.apache.org/" class="externalLink" title="Metron-Incubating">
+        Metron-Incubating</a>
+        </li>
+      <li class="divider ">/</li>
+            <li class="">
+                    <a href="../../index.html" title="Documentation">
+        Documentation</a>
+        </li>
+      <li class="divider ">/</li>
+        <li class="">Vagrant Deployment</li>
+        
+                
+                    
+                  <li id="publishDate" class="pull-right">Last Published: 2017-04-19</li> <li class="divider pull-right">|</li>
+              <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+            
+                            </ul>
+      </div>
+
+            
+      <div class="row-fluid">
+        <div id="leftColumn" class="span3">
+          <div class="well sidebar-nav">
+                
+                    
+                <ul class="nav nav-list">
+                    <li class="nav-header">User Documentation</li>
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+      <li>
+    
+                          <a href="../../index.html" title="Metron">
+          <i class="icon-chevron-down"></i>
+        Metron</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../Upgrading.html" title="Upgrading">
+          <i class="none"></i>
+        Upgrading</a>
+            </li>
+                                                                                                                                                      
+      <li>
+    
+                          <a href="../../metron-analytics/index.html" title="Analytics">
+          <i class="icon-chevron-right"></i>
+        Analytics</a>
+                  </li>
+                                                                                                                                                                                                                                                                                                                                                                            
+      <li>
+    
+                          <a href="../../metron-deployment/index.html" title="Deployment">
+          <i class="icon-chevron-down"></i>
+        Deployment</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2">
+          <i class="none"></i>
+        Amazon-ec2</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker">
+          <i class="none"></i>
+        Ansible-docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker">
+          <i class="none"></i>
+        Rpm-docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../metron-deployment/packaging/packer-build/index.html" title="Packer-build">
+          <i class="none"></i>
+        Packer-build</a>
+            </li>
+                                                                                                                                                
+      <li>
+    
+                          <a href="../../metron-deployment/roles/index.html" title="Roles">
+          <i class="icon-chevron-right"></i>
+        Roles</a>
+                  </li>
+                                                                                                                                                    
+      <li class="active">
+    
+            <a href="#"><i class="icon-chevron-down"></i>Vagrant</a>
+                  <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../metron-deployment/vagrant/Kerberos-setup.html" title="Kerberos-setup">
+          <i class="none"></i>
+        Kerberos-setup</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../metron-deployment/vagrant/codelab-platform/index.html" title="Codelab-platform">
+          <i class="none"></i>
+        Codelab-platform</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../metron-deployment/vagrant/fastcapa-test-platform/index.html" title="Fastcapa-test-platform">
+          <i class="none"></i>
+        Fastcapa-test-platform</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../metron-deployment/vagrant/full-dev-platform/index.html" title="Full-dev-platform">
+          <i class="none"></i>
+        Full-dev-platform</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../metron-deployment/vagrant/quick-dev-platform/index.html" title="Quick-dev-platform">
+          <i class="none"></i>
+        Quick-dev-platform</a>
+            </li>
+              </ul>
+        </li>
+              </ul>
+        </li>
+                      
+      <li>
+    
+                          <a href="../../metron-docker/index.html" title="Docker">
+          <i class="none"></i>
+        Docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../metron-interface/metron-config/index.html" title="Config">
+          <i class="none"></i>
+        Config</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../metron-interface/metron-rest/index.html" title="Rest">
+          <i class="none"></i>
+        Rest</a>
+            </li>
+                                                                                                                                                                                                                                                
+      <li>
+    
+                          <a href="../../metron-platform/index.html" title="Platform">
+          <i class="icon-chevron-right"></i>
+        Platform</a>
+                  </li>
+                                                                                                            
+      <li>
+    
+                          <a href="../../metron-sensors/index.html" title="Sensors">
+          <i class="icon-chevron-right"></i>
+        Sensors</a>
+                  </li>
+              </ul>
+        </li>
+            </ul>
+                
+                    
+                
+          <hr class="divider" />
+
+           <div id="poweredBy">
+                            <div class="clear"></div>
+                            <div class="clear"></div>
+                            <div class="clear"></div>
+                             <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+        <img class="builtBy" alt="Built by Maven" src="../../images/logos/maven-feather.png" />
+      </a>
+                  </div>
+          </div>
+        </div>
+        
+                
+        <div id="bodyColumn"  class="span9" >
+                                  
+            <h1>Vagrant Deployment</h1>
+<p><a name="Vagrant_Deployment"></a></p>
+
+<ul>
+  
+<li>Kerberos Setup</li>
+  
+<li>Codelab Platform</li>
+  
+<li>Fast CAPA Test Platform</li>
+  
+<li>Full Dev Platform</li>
+  
+<li>Quick Dev Platform</li>
+</ul>
+                  </div>
+            </div>
+          </div>
+
+    <hr/>
+
+    <footer>
+            <div class="container-fluid">
+              <div class="row span12">Copyright &copy;                   2017.
+          All Rights Reserved.      
+                    
+      </div>
+
+                          
+        
+                </div>
+    </footer>
+  </body>
+</html>

Added: dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-deployment/vagrant/quick-dev-platform/index.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-deployment/vagrant/quick-dev-platform/index.html (added)
+++ dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-deployment/vagrant/quick-dev-platform/index.html Thu Apr 20 05:38:46 2017
@@ -0,0 +1,348 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-19
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta name="Date-Revision-yyyymmdd" content="20170419" />
+    <meta http-equiv="Content-Language" content="en" />
+    <title>Metron &#x2013; Quick Development Platform</title>
+    <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.3.0.min.css" />
+    <link rel="stylesheet" href="../../../css/site.css" />
+    <link rel="stylesheet" href="../../../css/print.css" media="print" />
+
+      
+    <script type="text/javascript" src="../../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+                          
+        
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+          
+            </head>
+        <body class="topBarDisabled">
+          
+                
+                    
+    
+        <div class="container-fluid">
+          <div id="banner">
+        <div class="pull-left">
+                                    <a href="http://metron.incubator.apache.org/" id="bannerLeft">
+                                                                                                <img src="../../../images/metron-logo.png"  alt="Apache Metron - Incubating" width="148px" height="48px"/>
+                </a>
+                      </div>
+        <div class="pull-right">                  <a href="http://incubator.apache.org/" id="bannerRight">
+                                                                                                <img src="../../../images/ApacheIncubating_Logo.png"  alt="Apache Incubating" width="192px" height="48px"/>
+                </a>
+      </div>
+        <div class="clear"><hr/></div>
+      </div>
+
+      <div id="breadcrumbs">
+        <ul class="breadcrumb">
+                
+                    
+                              <li class="">
+                    <a href="http://www.apache.org" class="externalLink" title="Apache">
+        Apache</a>
+        </li>
+      <li class="divider ">/</li>
+            <li class="">
+                    <a href="http://metron.incubator.apache.org/" class="externalLink" title="Metron-Incubating">
+        Metron-Incubating</a>
+        </li>
+      <li class="divider ">/</li>
+            <li class="">
+                    <a href="../../../index.html" title="Documentation">
+        Documentation</a>
+        </li>
+      <li class="divider ">/</li>
+        <li class="">Quick Development Platform</li>
+        
+                
+                    
+                  <li id="publishDate" class="pull-right">Last Published: 2017-04-19</li> <li class="divider pull-right">|</li>
+              <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+            
+                            </ul>
+      </div>
+
+            
+      <div class="row-fluid">
+        <div id="leftColumn" class="span3">
+          <div class="well sidebar-nav">
+                
+                    
+                <ul class="nav nav-list">
+                    <li class="nav-header">User Documentation</li>
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+      <li>
+    
+                          <a href="../../../index.html" title="Metron">
+          <i class="icon-chevron-down"></i>
+        Metron</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../../Upgrading.html" title="Upgrading">
+          <i class="none"></i>
+        Upgrading</a>
+            </li>
+                                                                                                                                                      
+      <li>
+    
+                          <a href="../../../metron-analytics/index.html" title="Analytics">
+          <i class="icon-chevron-right"></i>
+        Analytics</a>
+                  </li>
+                                                                                                                                                                                                                                                                                                                                                                            
+      <li>
+    
+                          <a href="../../../metron-deployment/index.html" title="Deployment">
+          <i class="icon-chevron-down"></i>
+        Deployment</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2">
+          <i class="none"></i>
+        Amazon-ec2</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker">
+          <i class="none"></i>
+        Ansible-docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker">
+          <i class="none"></i>
+        Rpm-docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build">
+          <i class="none"></i>
+        Packer-build</a>
+            </li>
+                                                                                                                                                
+      <li>
+    
+                          <a href="../../../metron-deployment/roles/index.html" title="Roles">
+          <i class="icon-chevron-right"></i>
+        Roles</a>
+                  </li>
+                                                                                                                                                          
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/index.html" title="Vagrant">
+          <i class="icon-chevron-down"></i>
+        Vagrant</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/Kerberos-setup.html" title="Kerberos-setup">
+          <i class="none"></i>
+        Kerberos-setup</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/codelab-platform/index.html" title="Codelab-platform">
+          <i class="none"></i>
+        Codelab-platform</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/fastcapa-test-platform/index.html" title="Fastcapa-test-platform">
+          <i class="none"></i>
+        Fastcapa-test-platform</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/full-dev-platform/index.html" title="Full-dev-platform">
+          <i class="none"></i>
+        Full-dev-platform</a>
+            </li>
+                      
+      <li class="active">
+    
+            <a href="#"><i class="none"></i>Quick-dev-platform</a>
+          </li>
+              </ul>
+        </li>
+              </ul>
+        </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-docker/index.html" title="Docker">
+          <i class="none"></i>
+        Docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-interface/metron-config/index.html" title="Config">
+          <i class="none"></i>
+        Config</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-interface/metron-rest/index.html" title="Rest">
+          <i class="none"></i>
+        Rest</a>
+            </li>
+                                                                                                                                                                                                                                                
+      <li>
+    
+                          <a href="../../../metron-platform/index.html" title="Platform">
+          <i class="icon-chevron-right"></i>
+        Platform</a>
+                  </li>
+                                                                                                            
+      <li>
+    
+                          <a href="../../../metron-sensors/index.html" title="Sensors">
+          <i class="icon-chevron-right"></i>
+        Sensors</a>
+                  </li>
+              </ul>
+        </li>
+            </ul>
+                
+                    
+                
+          <hr class="divider" />
+
+           <div id="poweredBy">
+                            <div class="clear"></div>
+                            <div class="clear"></div>
+                            <div class="clear"></div>
+                             <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+        <img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" />
+      </a>
+                  </div>
+          </div>
+        </div>
+        
+                
+        <div id="bodyColumn"  class="span9" >
+                                  
+            <h1>Quick Development Platform</h1>
+<p>This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running on Virtualbox. </p>
+<p>This image is designed for quick deployment of a single node Metron cluster running on Virtualbox. This platform is ideal for use by Metron developers. It uses a base image that has been pre-loaded with Ambari and HDP.</p>
+<p>Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. To that end the vagrant file configuration defaults to disabling solr and yaf.</p>
+<div class="section">
+<h2><a name="Getting_Started"></a>Getting Started</h2>
+<div class="section">
+<h3><a name="Prerequisites"></a>Prerequisites</h3>
+<p>As with the Full Development Platform (<tt>metron-deployment/vagrant/full-dev-platform</tt>), the computer used to deploy Apache Metron will need the following components installed.</p>
+
+<ul>
+  
+<li><a class="externalLink" href="https://github.com/ansible/ansible">Ansible</a> (2.0.0.2 or 2.2.2.0)</li>
+  
+<li><a class="externalLink" href="https://www.vagrantup.com">Vagrant</a> 1.8.1</li>
+  
+<li><a class="externalLink" href="https://virtualbox.org">Virtualbox</a> 5.0.16</li>
+  
+<li>Python 2.7.11</li>
+  
+<li>Maven 3.3.9</li>
+</ul></div>
+<div class="section">
+<h3><a name="Deploy_Metron"></a>Deploy Metron</h3>
+
+<ol style="list-style-type: decimal">
+  
+<li>
+<p>Build Metron</p>
+  
+<div class="source">
+<div class="source">
+<pre>  cd incubator-metron
+  mvn clean package -DskipTests
+</pre></div></div></li>
+  
+<li>
+<p>Install Vagrant Hostmanager.</p>
+  
+<div class="source">
+<div class="source">
+<pre>  vagrant plugin install vagrant-hostmanager
+</pre></div></div></li>
+  
+<li>
+<p>Deploy Metron</p>
+  
+<div class="source">
+<div class="source">
+<pre>  cd metron-deployment/vagrant/quick-dev-platform
+  vagrant up
+</pre></div></div>
+<p>Should the process fail before completing the deployment, the following command will continue the deployment process without re-instantiating the host.</p>
+  
+<div class="source">
+<div class="source">
+<pre>  vagrant provision
+</pre></div></div></li>
+</ol></div>
+<div class="section">
+<h3><a name="Explore_Metron"></a>Explore Metron</h3>
+<p>Navigate to the following resources to explore your newly minted Apache Metron environment.</p>
+
+<ul>
+  
+<li><a class="externalLink" href="http://node1:8080">Ambari</a></li>
+  
+<li><a class="externalLink" href="http://node1:5000">Metron</a></li>
+  
+<li><a class="externalLink" href="http://node1:2812">Services</a></li>
+</ul></div>
+<div class="section">
+<h3><a name="Working_with_Metron"></a>Working with Metron</h3>
+<p>As you build out new capabilities for Metron, you will need to re-deploy the Storm topologies. To do so, first HALT the running Storm topologies and then run the following command.</p>
+
+<div class="source">
+<div class="source">
+<pre>./run_enrichment_role.sh
+</pre></div></div>
+<p>Connecting to the host through SSH is as simple as running the following command.</p>
+
+<div class="source">
+<div class="source">
+<pre>vagrant ssh
+</pre></div></div></div></div>
+                  </div>
+            </div>
+          </div>
+
+    <hr/>
+
+    <footer>
+            <div class="container-fluid">
+              <div class="row span12">Copyright &copy;                   2017.
+          All Rights Reserved.      
+                    
+      </div>
+
+                          
+        
+                </div>
+    </footer>
+  </body>
+</html>

Added: dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-docker/index.html
==============================================================================
--- dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-docker/index.html (added)
+++ dev/incubator/metron/0.4.0-RC1-incubating/book-site/metron-docker/index.html Thu Apr 20 05:38:46 2017
@@ -0,0 +1,488 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-04-19
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta name="Date-Revision-yyyymmdd" content="20170419" />
+    <meta http-equiv="Content-Language" content="en" />
+    <title>Metron &#x2013; Metron Docker</title>
+    <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" />
+    <link rel="stylesheet" href="../css/site.css" />
+    <link rel="stylesheet" href="../css/print.css" media="print" />
+
+      
+    <script type="text/javascript" src="../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+                          
+        
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+          
+            </head>
+        <body class="topBarDisabled">
+          
+                
+                    
+    
+        <div class="container-fluid">
+          <div id="banner">
+        <div class="pull-left">
+                                    <a href="http://metron.incubator.apache.org/" id="bannerLeft">
+                                                                                                <img src="../images/metron-logo.png"  alt="Apache Metron - Incubating" width="148px" height="48px"/>
+                </a>
+                      </div>
+        <div class="pull-right">                  <a href="http://incubator.apache.org/" id="bannerRight">
+                                                                                                <img src="../images/ApacheIncubating_Logo.png"  alt="Apache Incubating" width="192px" height="48px"/>
+                </a>
+      </div>
+        <div class="clear"><hr/></div>
+      </div>
+
+      <div id="breadcrumbs">
+        <ul class="breadcrumb">
+                
+                    
+                              <li class="">
+                    <a href="http://www.apache.org" class="externalLink" title="Apache">
+        Apache</a>
+        </li>
+      <li class="divider ">/</li>
+            <li class="">
+                    <a href="http://metron.incubator.apache.org/" class="externalLink" title="Metron-Incubating">
+        Metron-Incubating</a>
+        </li>
+      <li class="divider ">/</li>
+            <li class="">
+                    <a href="../index.html" title="Documentation">
+        Documentation</a>
+        </li>
+      <li class="divider ">/</li>
+        <li class="">Metron Docker</li>
+        
+                
+                    
+                  <li id="publishDate" class="pull-right">Last Published: 2017-04-19</li> <li class="divider pull-right">|</li>
+              <li id="projectVersion" class="pull-right">Version: 0.4.0</li>
+            
+                            </ul>
+      </div>
+
+            
+      <div class="row-fluid">
+        <div id="leftColumn" class="span3">
+          <div class="well sidebar-nav">
+                
+                    
+                <ul class="nav nav-list">
+                    <li class="nav-header">User Documentation</li>
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
+      <li>
+    
+                          <a href="../index.html" title="Metron">
+          <i class="icon-chevron-down"></i>
+        Metron</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../Upgrading.html" title="Upgrading">
+          <i class="none"></i>
+        Upgrading</a>
+            </li>
+                                                                                                                                                      
+      <li>
+    
+                          <a href="../metron-analytics/index.html" title="Analytics">
+          <i class="icon-chevron-right"></i>
+        Analytics</a>
+                  </li>
+                                                                                                                                                                                                                                                                                                                                                                  
+      <li>
+    
+                          <a href="../metron-deployment/index.html" title="Deployment">
+          <i class="icon-chevron-right"></i>
+        Deployment</a>
+                  </li>
+                      
+      <li class="active">
+    
+            <a href="#"><i class="none"></i>Docker</a>
+          </li>
+                      
+      <li>
+    
+                          <a href="../metron-interface/metron-config/index.html" title="Config">
+          <i class="none"></i>
+        Config</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../metron-interface/metron-rest/index.html" title="Rest">
+          <i class="none"></i>
+        Rest</a>
+            </li>
+                                                                                                                                                                                                                                                
+      <li>
+    
+                          <a href="../metron-platform/index.html" title="Platform">
+          <i class="icon-chevron-right"></i>
+        Platform</a>
+                  </li>
+                                                                                                            
+      <li>
+    
+                          <a href="../metron-sensors/index.html" title="Sensors">
+          <i class="icon-chevron-right"></i>
+        Sensors</a>
+                  </li>
+              </ul>
+        </li>
+            </ul>
+                
+                    
+                
+          <hr class="divider" />
+
+           <div id="poweredBy">
+                            <div class="clear"></div>
+                            <div class="clear"></div>
+                            <div class="clear"></div>
+                             <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+        <img class="builtBy" alt="Built by Maven" src="../images/logos/maven-feather.png" />
+      </a>
+                  </div>
+          </div>
+        </div>
+        
+                
+        <div id="bodyColumn"  class="span9" >
+                                  
+            <h1>Metron Docker</h1>
+<p><a name="Metron_Docker"></a></p>
+<p>Metron Docker is a <a class="externalLink" href="https://docs.docker.com/compose/overview/">Docker Compose</a> application that is intended for development and integration testing of Metron. Use this instead of Vagrant when:</p>
+
+<ul>
+  
+<li>You want an environment that can be built and spun up quickly</li>
+  
+<li>You need to frequently rebuild and restart services</li>
+  
+<li>You only need to test, troubleshoot or develop against a subset of services</li>
+</ul>
+<p>Metron Docker includes these images that have been customized for Metron:</p>
+
+<ul>
+  
+<li>Kafka (with Zookeeper)</li>
+  
+<li>HBase</li>
+  
+<li>Storm (with all topologies deployed)</li>
+  
+<li>Elasticsearch</li>
+  
+<li>Kibana</li>
+  
+<li>HDFS</li>
+</ul>
+<div class="section">
+<h2><a name="Setup"></a>Setup</h2>
+<p>Install <a class="externalLink" href="https://docs.docker.com/docker-for-mac/">Docker for Mac</a> or <a class="externalLink" href="https://docs.docker.com/docker-for-windows/">Docker for Windows</a>. The following versions have been tested:</p>
+
+<ul>
+  
+<li>Docker version 1.12.0</li>
+  
+<li>docker-machine version 0.8.0</li>
+  
+<li>docker-compose version 1.8.0</li>
+</ul>
+<p>Build Metron from the top level directory with:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_HOME
+$ mvn clean install -DskipTests
+</pre></div></div>
+<p>You are welcome to use an existing Docker host but we prefer one with more resources. You can create one of those with this script:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ export METRON_DOCKER_HOME=$METRON_HOME/metron-docker
+$ cd $METRON_DOCKER_HOME &amp;&amp; ./scripts/create-docker-machine.sh
+</pre></div></div>
+<p>This will create a host called &#x201c;metron-machine&#x201d;. Anytime you want to run Docker commands against this host, make sure you run this first to set the Docker environment variables:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ eval &quot;$(docker-machine env metron-machine)&quot;
+</pre></div></div>
+<p>If you wish to use a local docker-engine install, please set an environment variable BROKER_IP_ADDR to the IP address of your host machine. This cannot be the loopback address.</p></div>
+<div class="section">
+<h2><a name="Usage"></a>Usage</h2>
+<p>Navigate to the compose application root:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_DOCKER_HOME/compose/
+</pre></div></div>
+<p>The Metron Docker environment lifecycle is controlled by the <a class="externalLink" href="https://docs.docker.com/compose/reference/overview/">docker-compose</a> command. The service names can be found in the docker-compose.yml file. For example, to build and start the environment run this command:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ eval &quot;$(docker-machine env metron-machine)&quot;
+$ docker-compose up -d
+</pre></div></div>
+<p>After all services have started list the containers and ensure their status is &#x2018;Up&#x2019;:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker ps --format 'table {{.Names}}\t{{.Status}}'
+NAMES                    STATUS
+metron_storm_1           Up 5 minutes
+metron_hbase_1           Up 5 minutes
+metron_kibana_1          Up 5 minutes
+metron_kafkazk_1         Up 5 minutes
+metron_elasticsearch_1   Up 5 minutes
+</pre></div></div>
+<p>Various services are exposed through http on the Docker host. Get the host ip from the URL property:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-machine ls
+NAME             ACTIVE   DRIVER       STATE     URL                         SWARM   DOCKER    ERRORS
+metron-machine   *        virtualbox   Running   tcp://192.168.99.100:2376           v1.12.5
+</pre></div></div>
+<p>Then, assuming a host ip of <tt>192.168.99.100</tt>, the UIs and APIs are available at:</p>
+
+<ul>
+  
+<li>Storm - <a class="externalLink" href="http://192.168.99.100:8080/">http://192.168.99.100:8080/</a></li>
+  
+<li>HBase - <a class="externalLink" href="http://192.168.99.100:16010/">http://192.168.99.100:16010/</a></li>
+  
+<li>Elasticsearch - <a class="externalLink" href="http://192.168.99.100:9200/_plugin/head/">http://192.168.99.100:9200/_plugin/head/</a></li>
+  
+<li>Kibana - <a class="externalLink" href="http://192.168.99.100:5601/">http://192.168.99.100:5601/</a></li>
+  
+<li>HDFS (Namenode) - <a class="externalLink" href="http://192.168.99.100:50070/">http://192.168.99.100:50070/</a></li>
+</ul>
+<p>The Storm logs can be useful when troubleshooting topologies. They can be found on the Storm container in <tt>/usr/share/apache-storm/logs</tt>.</p>
+<p>When done using the machine, shut it down with:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-compose down
+</pre></div></div></div>
+<div class="section">
+<h2><a name="Examples"></a>Examples</h2>
+
+<ul>
+  
+<li><a href="#Deploy_a_new_parser_class">Deploy a new parser class</a></li>
+  
+<li><a href="#Connect_to_a_container">Connect to a container</a></li>
+  
+<li><a href="create-a-sensor-from-sample-data/index.html">Create a sensor from sample data</a></li>
+  
+<li><a href="upload-configs-to-zookeeper/index.html">Upload configs to Zookeeper</a></li>
+  
+<li><a href="manage-a-topology/index.html">Manage a topology</a></li>
+  
+<li><a href="run-sensor-data-end-to-end/index.html">Run sensor data end to end</a></li>
+</ul>
+<div class="section">
+<h3><a name="Deploy_a_new_parser_class"></a>Deploy a new parser class</h3>
+<p>After adding a new parser to metron-parsers, build Metron from the top level directory:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_HOME
+$ mvn clean install -DskipTests
+</pre></div></div>
+<p>Then run these commands to redeploy the parsers to the Storm image:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_DOCKER_HOME/compose
+$ docker-compose down
+$ docker-compose build storm
+$ docker-compose up -d
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Connect_to_a_container"></a>Connect to a container</h3>
+<p>Suppose there is a problem with Kafka and the logs are needed for further investigation. Run this command to connect and explore the running Kafka container:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_DOCKER_HOME/compose
+$ docker-compose exec kafkazk bash
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Create_a_sensor_from_sample_data"></a>Create a sensor from sample data</h3>
+<p>A tool for producing test data in Kafka is included with the Kafka/Zookeeper image. It loops through lines in a test data file and outputs them to Kafka at the desired frequency. Create a test data file in <tt>./kafkazk/data/</tt> and rebuild the Kafka/Zookeeper image:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_DOCKER_HOME/compose
+$ printf 'first test data\nsecond test data\nthird test data\n' &gt; ./kafkazk/data/TestData.txt
+$ docker-compose down
+$ docker-compose build kafkazk
+$ docker-compose up -d
+</pre></div></div>
+<p>This will deploy the test data file to the Kafka/Zookeeper container. Now that data can be streamed to a Kafka topic:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-compose exec kafkazk ./bin/produce-data.sh
+Usage:  produce-data.sh data_path topic [message_delay_in_seconds]
+
+# Stream data in TestData.txt to the 'test' Kafka topic at a frequency of 5 seconds (default is 1 second)
+$ docker-compose exec kafkazk ./bin/produce-data.sh /data/TestData.txt test 5 
+</pre></div></div>
+<p>The Kafka/Zookeeper image comes with sample Bro and Squid data:</p>
+
+<div class="source">
+<div class="source">
+<pre># Stream Bro test data every 1 second
+$ docker-compose exec kafkazk ./bin/produce-data.sh /data/BroExampleOutput.txt bro
+
+# Stream Squid test data every 0.1 seconds
+$ docker-compose exec kafkazk ./bin/produce-data.sh /data/SquidExampleOutput.txt squid 0.1
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Upload_configs_to_Zookeeper"></a>Upload configs to Zookeeper</h3>
+<p>Parser configs and a global config configured for this Docker environment are included with the Kafka/Zookeeper image. Load them with:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-compose exec kafkazk bash
+# $METRON_HOME/bin/zk_load_configs.sh -z localhost:2181 -m PUSH -i $METRON_HOME/config/zookeeper
+# exit
+</pre></div></div>
+<p>Dump out the configs with:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-compose exec kafkazk bash
+# $METRON_HOME/bin/zk_load_configs.sh -z localhost:2181 -m DUMP
+# exit
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Manage_a_topology"></a>Manage a topology</h3>
+<p>The Storm image comes with a script to easily start parser topologies:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec storm ./bin/start_docker_parser_topology.sh sensor_name
+</pre></div></div>
+<p>The enrichment topology can be started with:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec storm ./bin/start_enrichment_topology.sh
+</pre></div></div>
+<p>The indexing topology can be started with:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec storm ./bin/start_elasticsearch_topology.sh
+</pre></div></div>
+<p>Topologies can be stopped using the Storm CLI. For example, stop the enrichment topology with:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec storm storm kill enrichments -w 0
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Run_sensor_data_end_to_end"></a>Run sensor data end to end</h3>
+<p>First ensure configs were uploaded as described in the previous example. Then start a sensor and leave it running:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ cd $METRON_DOCKER_HOME/compose
+$ docker-compose exec kafkazk ./bin/produce-data.sh /data/BroExampleOutput.txt bro
+</pre></div></div>
+<p>Open a separate console session and verify the sensor is running by consuming a message from Kafka:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ export METRON_DOCKER_HOME=$METRON_HOME/metron-docker
+$ cd $METRON_DOCKER_HOME/compose
+$ docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic bro
+</pre></div></div>
+<p>A new message should be printed every second. Now kill the consumer and start the Bro parser topology:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-compose exec storm ./bin/start_docker_parser_topology.sh bro
+</pre></div></div>
+<p>Bro data should be flowing through the bro parser topology and into the Kafka enrichments topic. The enrichments topic should be created automatically:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-compose exec kafkazk ./bin/kafka-topics.sh --zookeeper localhost:2181 --list
+bro
+enrichments
+indexing
+</pre></div></div>
+<p>Verify parsed Bro data is in the Kafka enrichments topic:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic enrichments
+</pre></div></div>
+<p>Now start the enrichment topology:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec storm ./bin/start_enrichment_topology.sh
+</pre></div></div>
+<p>Parsed Bro data should be flowing through the enrichment topology and into the Kafka indexing topic. Verify enriched Bro data is in the Kafka indexing topic:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec kafkazk ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic indexing
+</pre></div></div>
+<p>Now start the indexing topology:</p>
+
+<div class="source">
+<div class="source">
+<pre>docker-compose exec storm ./bin/start_elasticsearch_topology.sh
+</pre></div></div>
+<p>Enriched Bro data should now be present in the Elasticsearch container:</p>
+
+<div class="source">
+<div class="source">
+<pre>$ docker-machine ls
+NAME             ACTIVE   DRIVER       STATE     URL                         SWARM   DOCKER    ERRORS
+metron-machine   *        virtualbox   Running   tcp://192.168.99.100:2376           v1.12.5
+
+$ curl -XGET http://192.168.99.100:9200/_cat/indices?v
+health status index                   pri rep docs.count docs.deleted store.size pri.store.size
+yellow open   .kibana                   1   1          1            0      3.1kb          3.1kb
+yellow open   bro_index_2016.12.19.18   5   1        180            0      475kb          475kb
+</pre></div></div></div></div>
+                  </div>
+            </div>
+          </div>
+
+    <hr/>
+
+    <footer>
+            <div class="container-fluid">
+              <div class="row span12">Copyright &copy;                   2017.
+          All Rights Reserved.      
+                    
+      </div>
+
+                          
+        
+                </div>
+    </footer>
+  </body>
+</html>



Mime
View raw message