metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject [10/12] metron git commit: METRON-1191 Sync-ing asf-site from the generated code on master
Date Tue, 19 Sep 2017 18:59:20 GMT
http://git-wip-us.apache.org/repos/asf/metron/blob/53295c5a/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html
----------------------------------------------------------------------
diff --git a/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html b/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html
new file mode 100644
index 0000000..62128e1
--- /dev/null
+++ b/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html
@@ -0,0 +1,1648 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2017-09-15
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta name="Date-Revision-yyyymmdd" content="20170915" />
+    <meta http-equiv="Content-Language" content="en" />
+    <title>Metron &#x2013; </title>
+    <link rel="stylesheet" href="../../../css/apache-maven-fluido-1.3.0.min.css" />
+    <link rel="stylesheet" href="../../../css/site.css" />
+    <link rel="stylesheet" href="../../../css/print.css" media="print" />
+
+      
+    <script type="text/javascript" src="../../../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+                          
+        
+<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
+          
+            </head>
+        <body class="topBarDisabled">
+          
+                
+                    
+    
+        <div class="container-fluid">
+          <div id="banner">
+        <div class="pull-left">
+                                    <a href="http://metron.apache.org/" id="bannerLeft">
+                                                                                                <img src="../../../images/metron-logo.png"  alt="Apache Metron" width="148px" height="48px"/>
+                </a>
+                      </div>
+        <div class="pull-right">  </div>
+        <div class="clear"><hr/></div>
+      </div>
+
+      <div id="breadcrumbs">
+        <ul class="breadcrumb">
+                
+                    
+                              <li class="">
+                    <a href="http://www.apache.org" class="externalLink" title="Apache">
+        Apache</a>
+        </li>
+      <li class="divider ">/</li>
+            <li class="">
+                    <a href="http://metron.apache.org/" class="externalLink" title="Metron">
+        Metron</a>
+        </li>
+      <li class="divider ">/</li>
+            <li class="">
+                    <a href="../../../index.html" title="Documentation">
+        Documentation</a>
+        </li>
+      <li class="divider ">/</li>
+        <li class=""></li>
+        
+                
+                    
+                  <li id="publishDate" class="pull-right">Last Published: 2017-09-15</li> <li class="divider pull-right">|</li>
+              <li id="projectVersion" class="pull-right">Version: 0.4.1</li>
+            
+                            </ul>
+      </div>
+
+            
+      <div class="row-fluid">
+        <div id="leftColumn" class="span3">
+          <div class="well sidebar-nav">
+                
+                    
+                <ul class="nav nav-list">
+                    <li class="nav-header">User Documentation</li>
+                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     
                                                                          
+      <li>
+    
+                          <a href="../../../index.html" title="Metron">
+          <i class="icon-chevron-down"></i>
+        Metron</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../../Upgrading.html" title="Upgrading">
+          <i class="none"></i>
+        Upgrading</a>
+            </li>
+                                                                                                                                                      
+      <li>
+    
+                          <a href="../../../metron-analytics/index.html" title="Analytics">
+          <i class="icon-chevron-right"></i>
+        Analytics</a>
+                  </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-contrib/metron-docker/index.html" title="Docker">
+          <i class="none"></i>
+        Docker</a>
+            </li>
+                                                                                                                                                                                                                                                                                                                                                                                                                                                          
+      <li>
+    
+                          <a href="../../../metron-deployment/index.html" title="Deployment">
+          <i class="icon-chevron-down"></i>
+        Deployment</a>
+                    <ul class="nav nav-list">
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/Kerberos-ambari-setup.html" title="Kerberos-ambari-setup">
+          <i class="none"></i>
+        Kerberos-ambari-setup</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/Kerberos-manual-setup.html" title="Kerberos-manual-setup">
+          <i class="none"></i>
+        Kerberos-manual-setup</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/amazon-ec2/index.html" title="Amazon-ec2">
+          <i class="none"></i>
+        Amazon-ec2</a>
+            </li>
+                                                                                  
+      <li>
+    
+                          <a href="../../../metron-deployment/other-examples/index.html" title="Other-examples">
+          <i class="icon-chevron-down"></i>
+        Other-examples</a>
+                    <ul class="nav nav-list">
+                      
+      <li class="active">
+    
+            <a href="#"><i class="none"></i>Manual_Install_CentOS6</a>
+          </li>
+              </ul>
+        </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/ambari/index.html" title="Ambari">
+          <i class="none"></i>
+        Ambari</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/docker/ansible-docker/index.html" title="Ansible-docker">
+          <i class="none"></i>
+        Ansible-docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/docker/rpm-docker/index.html" title="Rpm-docker">
+          <i class="none"></i>
+        Rpm-docker</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-deployment/packaging/packer-build/index.html" title="Packer-build">
+          <i class="none"></i>
+        Packer-build</a>
+            </li>
+                                                                                                                                                
+      <li>
+    
+                          <a href="../../../metron-deployment/roles/index.html" title="Roles">
+          <i class="icon-chevron-right"></i>
+        Roles</a>
+                  </li>
+                                                                                                                              
+      <li>
+    
+                          <a href="../../../metron-deployment/vagrant/index.html" title="Vagrant">
+          <i class="icon-chevron-right"></i>
+        Vagrant</a>
+                  </li>
+              </ul>
+        </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-interface/metron-alerts/index.html" title="Alerts">
+          <i class="none"></i>
+        Alerts</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-interface/metron-config/index.html" title="Config">
+          <i class="none"></i>
+        Config</a>
+            </li>
+                      
+      <li>
+    
+                          <a href="../../../metron-interface/metron-rest/index.html" title="Rest">
+          <i class="none"></i>
+        Rest</a>
+            </li>
+                                                                                                                                                                                                                                                                  
+      <li>
+    
+                          <a href="../../../metron-platform/index.html" title="Platform">
+          <i class="icon-chevron-right"></i>
+        Platform</a>
+                  </li>
+                                                                                                            
+      <li>
+    
+                          <a href="../../../metron-sensors/index.html" title="Sensors">
+          <i class="icon-chevron-right"></i>
+        Sensors</a>
+                  </li>
+                                                                        
+      <li>
+    
+                          <a href="../../../metron-stellar/stellar-common/index.html" title="Stellar-common">
+          <i class="icon-chevron-right"></i>
+        Stellar-common</a>
+                  </li>
+                                                                        
+      <li>
+    
+                          <a href="../../../use-cases/index.html" title="Use-cases">
+          <i class="icon-chevron-right"></i>
+        Use-cases</a>
+                  </li>
+              </ul>
+        </li>
+            </ul>
+                
+                    
+                
+          <hr class="divider" />
+
+           <div id="poweredBy">
+                            <div class="clear"></div>
+                            <div class="clear"></div>
+                            <div class="clear"></div>
+                             <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+        <img class="builtBy" alt="Built by Maven" src="../../../images/logos/maven-feather.png" />
+      </a>
+                  </div>
+          </div>
+        </div>
+        
+                
+        <div id="bodyColumn"  class="span9" >
+                                  
+            <div class="section">
+<h2><a name="Metron_0.4.0_with_HDP_2.5_bare-metal_install_on_Centos_6_with_MariaDB_for_Metron_REST:"></a>Metron 0.4.0 with HDP 2.5 bare-metal install on Centos 6 with MariaDB for Metron REST:</h2>
+<div class="section">
+<h3><a name="Introduction"></a>Introduction</h3>
+<p>We will be installing Metron 0.4.0 with HDP 2.5 on CentOS 6. We will also install MariaDB as a database for Metron REST. Additionally, we&#x2019;ll also install Apache NiFi. I installed Metron in a test environment with 3 VMs to try it out as well as a single node. I&#x2019;ll try to write this guide so that the necessary steps can easily be adapted for other environments.</p></div>
+<div class="section">
+<h3><a name="Environment"></a>Environment</h3>
+
+<ul>
+  
+<li>
+<p>Single node: 4 CPUs, 16 GB RAM.</p></li>
+  
+<li>
+<p>Multiple nodes:</p>
+  
+<ul>
+    
+<li>3 VMs, 2 CPUs per VM and 8 GB RAM per VM.</li>
+    
+<li>Hosts: 10.10.10.1 node1 10.10.10.2 node2 10.10.10.3 node3</li>
+  </ul></li>
+</ul></div>
+<div class="section">
+<h3><a name="Prerequisites:"></a>Prerequisites:</h3>
+
+<ul>
+  
+<li>
+<p>CentOS 6</p></li>
+  
+<li>
+<p>Add the epel repository and install tmux, vim &amp; htop. Installing these utilities is not strictly necessary, but I install these by default for potential troubleshooting &amp; editing of files locally):</p></li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># yum install epel-release -y
+# yum update -y
+# yum install vim tmux htop -y
+</pre></div></div>
+
+<ul>
+  
+<li>Set up passwordless SSH between our nodes: If passwordless ssh has not yet been set up within the cluster, then in main node generate key:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># cat /dev/zero | ssh-keygen -q -N &quot;&quot; 2&gt;/dev/null
+</pre></div></div>
+<p>If you&#x2019;re not installing on a single node, add this newly generated key to all the slave nodes:</p>
+
+<div class="source">
+<div class="source">
+<pre>ssh-copy-id -i ~/.ssh/id_rsa.pub &lt;replace_with_node_ip&gt;
+</pre></div></div>
+<p><i>Side note:</i> You might have to adapt your sshd_config file and add &#x201c;PermitRootLogin yes&#x201d; amongst other parameters if you want passwordless root access, but that&#x2019;s outside the scope of this document.</p>
+
+<ul>
+  
+<li>Increase limits for ElasticSearch and Storm on nodes where you will be installing them (if you don&#x2019;t know, increase it everywhere):</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># echo -e &quot;elasticsearch - memlock unlimited\nstorm - nproc 257597&quot; &gt;&gt; /etc/security/limits.conf
+</pre></div></div>
+
+<ul>
+  
+<li>Adjust limits to secure level (<a class="externalLink" href="https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.0/bk_installing_manually_book/content/ref-729d1fb0-6d1b-459f-a18a-b5eba4540ab5.1.html">link</a>):</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># ulimit -n 32768
+# ulimit -u 65536
+# echo -e &quot;* - nofile 32768\n* - nproc 65536&quot; &gt;&gt; /etc/security/limits.conf
+</pre></div></div>
+
+<ul>
+  
+<li>Disable IPv6, leaving it enabled may force service to bind to IPv6 addresses only and thus resulting in inability to connect to it (<a class="externalLink" href="https://wiki.centos.org/FAQ/CentOS6#head-d47139912868bcb9d754441ecb6a8a10d41781df">source link</a>): Disable for the running system:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># sysctl -w net.ipv6.conf.all.disable_ipv6=1
+# sysctl -w net.ipv6.conf.default.disable_ipv6=1
+or
+# echo 1 &gt; /proc/sys/net/ipv6/conf/all/disable_ipv6
+# echo 1 &gt; /proc/sys/net/ipv6/conf/default/disable_ipv6
+</pre></div></div>
+<p>To survive a reboot: Add: net.ipv6.conf.all.disable_ipv6 = 1 net.ipv6.conf.default.disable_ipv6 = 1 To: /etc/sysctl.conf</p>
+
+<div class="source">
+<div class="source">
+<pre># echo -e &quot;\n# Disable IPv6\nnet.ipv6.conf.all.disable_ipv6 = 1\nnet.ipv6.conf.default.disable_ipv6 = 1&quot; &gt;&gt; /etc/sysctl.conf
+</pre></div></div>
+
+<ul>
+  
+<li>Disable Transparent Hugepage. Add &#x201c;transparent_hugepage=never&#x201d; to the end of the kernel line in /boot/grub/grub.conf and reboot. (Ambari demands it, do we need to comply?):</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre>Add &quot;transparent_hugepage=never&quot; in the kernel line after &quot;quiet:
+&quot;kernel /vmlinuz-2.6.32-696.3.1.el6.x86_64 ro root=/dev/mapper/vg_centos6-lv_root rd_NO_LUKS LANG=en_US.UTF-8 rd_LVM_LV=vg_centos6/lv_swap rd_NO_MD SYSFONT=latarcyrheb-sun16 crashkernel=128M rd_LVM_LV=vg_centos6/lv_root  KEYBOARDTYPE=pc KEYTABLE=us rd_NO_DM rhgb quiet&quot;
+becomes:
+&quot;kernel /vmlinuz-2.6.32-696.3.1.el6.x86_64 ro root=/dev/mapper/vg_centos6-lv_root rd_NO_LUKS LANG=en_US.UTF-8 rd_LVM_LV=vg_centos6/lv_swap rd_NO_MD SYSFONT=latarcyrheb-sun16 crashkernel=128M rd_LVM_LV=vg_centos6/lv_root  KEYBOARDTYPE=pc KEYTABLE=us rd_NO_DM rhgb quiet transparent_hugepage=never&quot;
+Afterwards, run:
+# grub-install /dev/sda
+
+</pre></div></div>
+<p>After reboot check that changes were applied (make sure that word &#x201c;never&#x201d; is selected in square-brackets):</p>
+
+<div class="source">
+<div class="source">
+<pre># cat /sys/kernel/mm/transparent_hugepage/enabled
+always madvise [never]
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Metron_install_pre-preparation:"></a>Metron install pre-preparation:</h3>
+
+<ul>
+  
+<li>On all nodes Install pre-requisites for Ambari:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># yum install git wget curl rpm tar unzip bzip2 wget createrepo yum-utils ntp python-pip psutils python-psutil ntp libffi-devel gcc openssl-devel -y
+# pip install --upgrade pip
+# pip install requests urllib
+# pip install --upgrade setuptools
+</pre></div></div>
+
+<ul>
+  
+<li>Install Maven on main node and on Metron node install java 1.8 (if you don&#x2019;t know which it is, install it everywhere):</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># yum install java-1.8.0-openjdk java-1.8.0-openjdk-devel -y
+</pre></div></div>
+
+<ul>
+  
+<li>Set path to Java 8 if it does not exist:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># export JAVA_HOME=$(readlink -f /usr/bin/java | sed &quot;s_/jre/bin/java__&quot;)
+</pre></div></div>
+
+<ul>
+  
+<li>Save export for future reboots:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># echo 'export JAVA_HOME=$(readlink -f /usr/bin/java | sed &quot;s_/jre/bin/java__&quot;)' &gt; /etc/profile.d/java_18.sh
+# chmod +x /etc/profile.d/java_18.sh
+# source /etc/profile.d/java_18.sh
+</pre></div></div>
+
+<ul>
+  
+<li>Download and install Maven:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># wget http://apache.volia.net/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
+# tar -zxf apache-maven-3.3.9-bin.tar.gz
+# mv apache-maven-3.3.9 /opt
+# PATH=/opt/apache-maven-3.3.9/bin:$PATH
+# echo 'export PATH=/opt/apache-maven-3.3.9/bin:$PATH' &gt; /etc/profile.d/maven.sh
+# chmod +x /etc/profile.d/maven.sh
+</pre></div></div>
+<p>Check whether Maven works:</p>
+
+<div class="source">
+<div class="source">
+<pre># source /etc/profile.d/maven.sh
+# mvn -V
+</pre></div></div>
+<p>You should see something similar to:</p>
+
+<div class="source">
+<div class="source">
+<pre>[root@base1 ~]# mvn -V
+Apache Maven 3.3.9 (bb52d8502b132ec0a5a3f4c09453c07478323dc5; 2015-11-10T08:41:47-08:00)
+Maven home: /opt/apache-maven-3.3.9
+Java version: 1.8.0_131, vendor: Oracle Corporation
+Java home: /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.131-3.b12.el7_3.x86_64/jre
+Default locale: en_US, platform encoding: UTF-8
+OS name: &quot;linux&quot;, version: &quot;3.10.0-514.16.1.el7.x86_64&quot;, arch: &quot;amd64&quot;, family: &quot;unix&quot;
+[INFO] Scanning for projects...
+[INFO] ------------------------------------------------------------------------
+[INFO] BUILD FAILURE
+[INFO] ------------------------------------------------------------------------
+[INFO] Total time: 0.083 s
+[INFO] Finished at: 2017-06-06T09:59:03-07:00
+[INFO] Final Memory: 13M/479M
+[INFO] ------------------------------------------------------------------------
+[ERROR] No goals have been specified for this build. You must specify a valid lifecycle phase or a goal in the format &lt;plugin-prefix&gt;:&lt;goal&gt; or &lt;plugin-group-id&gt;:&lt;plugin-artifact-id&gt;[:&lt;plugin-version&gt;]:&lt;goal&gt;. Available lifecycle phases are: validate, initialize, generate-sources, process-sources, generate-resources, process-resources, compile, process-classes, generate-test-sources, process-test-sources, generate-test-resources, process-test-resources, test-compile, process-test-classes, test, prepare-package, package, pre-integration-test, integration-test, post-integration-test, verify, install, deploy, pre-clean, clean, post-clean, pre-site, site, post-site, site-deploy. -&gt; [Help 1]
+[ERROR]
+[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch.
+[ERROR] Re-run Maven using the -X switch to enable full debug logging.
+[ERROR]
+[ERROR] For more information about the errors and possible solutions, please read the following articles:
+[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/NoGoalSpecifiedException
+[root@base1 ~]#
+</pre></div></div>
+
+<ul>
+  
+<li>On Ambari node install and enable docker (we will need it to build Metron mpack for Ambari):</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># yum install docker-io -y
+# service docker start
+</pre></div></div>
+
+<ul>
+  
+<li>Also on your build box, install npm. This is needed to build metron-config, part of the UI.</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># yum install npm -y
+</pre></div></div>
+
+<ul>
+  
+<li>Remove ipv4 &#x2018;localhost.localdomain&#x2019; from /etc/hosts</li>
+  
+<li>Remove ipv6 &#x2018;localhost.localdomain&#x2019; from /etc/hosts</li>
+  
+<li>Add &#x201c;127.0.0.1 localhost&#x201d; to /etc/hosts</li>
+  
+<li>
+<p>Install the database we will use for Metron REST:</p></li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># yum install mariadb-server mysql-connector-java -y
+</pre></div></div>
+
+<ul>
+  
+<li>Configure a user and a database for Metron REST: If you haven&#x2019;t run <tt>mysql_secure_installation</tt> after the database installation, do that first:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># service mysqld start
+# /sbin/chkconfig --add mysqld
+# /sbin/chkconfig --list mysqld
+# /sbin/chkconfig mysqld on
+# /sbin/chkconfig --list mysqld
+# mysql_secure_installation
+
+NOTE: RUNNING ALL PARTS OF THIS SCRIPT IS RECOMMENDED FOR ALL MySQL
+      SERVERS IN PRODUCTION USE!  PLEASE READ EACH STEP CAREFULLY!
+
+In order to log into MySQL to secure it, we'll need the current
+password for the root user.  If you've just installed MySQL, and
+you haven't set the root password yet, the password will be blank,
+so you should just press enter here.
+
+Enter current password for root (enter for none):
+OK, successfully used password, moving on...
+
+Setting the root password ensures that nobody can log into the MySQL
+root user without the proper authorisation.
+
+Set root password? [Y/n]
+New password:
+Re-enter new password:
+Password updated successfully!
+Reloading privilege tables..
+ ... Success!
+
+
+By default, a MySQL installation has an anonymous user, allowing anyone
+to log into MySQL without having to have a user account created for
+them.  This is intended only for testing, and to make the installation
+go a bit smoother.  You should remove them before moving into a
+production environment.
+
+Remove anonymous users? [Y/n] n
+ ... skipping.
+
+Normally, root should only be allowed to connect from 'localhost'.  This
+ensures that someone cannot guess at the root password from the network.
+
+Disallow root login remotely? [Y/n]
+ ... Success!
+By default, MySQL comes with a database named 'test' that anyone can
+access.  This is also intended only for testing, and should be removed
+before moving into a production environment.
+
+Remove test database and access to it? [Y/n]
+ - Dropping test database...
+ERROR 1008 (HY000) at line 1: Can't drop database 'test'; database doesn't exist
+ ... Failed!  Not critical, keep moving...
+ - Removing privileges on test database...
+ ... Success!
+
+Reloading the privilege tables will ensure that all changes made so far
+will take effect immediately.
+
+Reload privilege tables now? [Y/n]
+ ... Success!
+
+All done!  If you've completed all of the above steps, your MySQL
+installation should now be secure.
+
+Thanks for using MySQL!
+
+
+Cleaning up...
+#
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Build_Metron_code"></a>Build Metron code</h3>
+<p>Now we are going to start to building Metron. At the time of writing, Metron 0.4.0 was in the final stages of being released.</p>
+
+<ul>
+  
+<li>On the main node, clone the Metron repository:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># git clone https://github.com/apache/metron
+</pre></div></div>
+
+<ul>
+  
+<li>Build Metron with HDP 2.5 profile:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># cd metron
+# mvn clean package -DskipTests -T 2C -P HDP-2.5.0.0,mpack
+# cd metron-deployment/packaging/docker/rpm-docker
+# mvn clean install -DskipTests -PHDP-2.5.0.0
+</pre></div></div>
+<p>If for some reason, the rpm-docker fails with the message &#x201c;/bin/bash: ./build.sh: Permission denied&#x201d;, try disabling selinux (&#x201c;setenforce 0&#x201d;) and run &#x201c;mvn clean install -DskipTests -PHDP-2.5.0.0&#x201d; again.</p>
+
+<ul>
+  
+<li>On all nodes, create a localrepo directory and copy the RPMs from Ambari node there:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># mkdir /localrepo
+# cp -rp /root/metron/metron-deployment/packaging/docker/rpm-docker/RPMS/noarch/* /localrepo/
+# createrepo /localrepo
+</pre></div></div>
+<p>If you&#x2019;re doing a multi node install, also copy the packages to the other nodes:</p>
+
+<div class="source">
+<div class="source">
+<pre># scp /localrepo/* &lt;replace_with_node_ip&gt;:/localrepo/
+# createrepo /localrepo
+</pre></div></div>
+
+<ul>
+  
+<li>Make sure to run <tt>createrepo /localrepo</tt> on every node!</li>
+</ul>
+<p>Fetch &amp; create logrotate script for Hadoop Services:</p>
+
+<div class="source">
+<div class="source">
+<pre># wget -O /etc/logrotate.d/metron-ambari https://raw.githubusercontent.com/apache/metron/master/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml
+# sed -i 's/^  {{ hadoop_logrotate_frequency }}.*$/  daily/' /etc/logrotate.d/metron-ambari
+# sed -i 's/^  rotate {{ hadoop_logrotate_retention }}.*$/  rotate 30/' /etc/logrotate.d/metron-ambari
+# chmod 0644 /etc/logrotate.d/metron-ambari
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Ambari_2.4_with_HDP_2.5_install"></a>Ambari 2.4 with HDP 2.5 install</h3>
+<p>Inspired by: [http://docs.hortonworks.com/HDPDocuments/Ambari-2.4.1.0/bk_ambari-installation/content/ch_Getting_Ready.html]</p>
+
+<ul>
+  
+<li>Adjust limits to secure level (inspired by <a class="externalLink" href="https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.0/bk_installing_manually_book/content/ref-729d1fb0-6d1b-459f-a18a-b5eba4540ab5.1.html">link</a>):</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># ulimit -n 32768
+# ulimit -u 65536
+# echo -e &quot;* - nofile 32768\n* - nproc 65536&quot; &gt;&gt; /etc/security/limits.conf
+</pre></div></div>
+
+<ul>
+  
+<li>Enable time sync, disable firewall and SElinux:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># yum install ntp -y
+# service ntpd start
+# /sbin/chkconfig --add ntpd
+# /sbin/chkconfig --list ntpd
+# /sbin/chkconfig ntpd on
+# /sbin/chkconfig --list ntpd
+</pre></div></div>
+
+<ul>
+  
+<li>Disable firewall:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># service iptables save
+# service iptables stop
+# chkconfig iptables off
+</pre></div></div>
+
+<ul>
+  
+<li>Disable IPv6 firewall:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># service ip6tables save
+# service ip6tables stop
+# chkconfig ip6tables off
+</pre></div></div>
+
+<ul>
+  
+<li>Disable SElinux</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># setenforce 0 (=&gt; I know, but for the sake of simplicity, quickness &amp; testing, I've disabled selinux.)
+</pre></div></div>
+
+<ul>
+  
+<li>Make sure each node can resolve every other node&#x2019;s hostname or add hostname of each node to <tt>/etc/hosts</tt> on every node. For example add following lines in /etc/hosts of each node: 10.10.10.1 node1 10.10.10.2 node2 10.10.10.3 node3</li>
+</ul>
+<p>Where 10.10.10.1, 10.10.10.2 and 10.10.10.3 are the IPs of your nodes and node1, node2 and node3 are hostnames.</p>
+
+<ul>
+  
+<li>On main node download and setup Ambari repo (you may replace the &#x201c;2.4.2.0&#x201d; with a newer Ambari version number):</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># wget -nv http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.2.0/ambari.repo -O /etc/yum.repos.d/ambari.repo
+# yum update -y
+</pre></div></div>
+
+<ul>
+  
+<li>Check that it was added:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># yum repolist | grep ambari
+Updates-ambari-2.4.2.0   ambari-2.4.2.0 - Updates                             12
+</pre></div></div>
+
+<ul>
+  
+<li>Install and setup Ambari server:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># yum install ambari-server -y
+# ambari-server setup -s &amp;&amp; touch /etc/ambari-server/configured
+</pre></div></div>
+
+<ul>
+  
+<li>Add Metron service to Ambari by running mpack command (make sure to specify correct path to mpack in &#x2013;mpack=):</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># ambari-server install-mpack --mpack=/root/metron/metron-deployment/packaging/ambari/metron-mpack/target/metron_mpack-0.4.0.0.tar.gz --verbose
+</pre></div></div>
+
+<ul>
+  
+<li>Start Ambari:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># ambari-server start
+</pre></div></div>
+
+<ul>
+  
+<li>Access the Ambari UI by going to the following URL in a web browser: <tt>http://&lt;replace_with_master_node_ip&gt;:8080/</tt>. You can use admin/admin as username/password. Start the Install Wizard.</li>
+</ul>
+<p><b>Get Started page:</b> Enter any desired cluster name.</p>
+<p><b>Select Version:</b> Make sure &#x201c;Public Repository&#x201d; is checked. You should also see the <tt>/localrepo</tt> directory listed.</p>
+<p><b>Install Options</b>: Specify hostnames of your nodes where Ambari cluster should be installed (all the ones you have specified in /etc/hosts) in Target Hosts. Copy content of the main node private key (/root/.ssh/id_rsa) in &#x201c;Host Registration Information&#x201d;. If you receive warning like below, ignore it and click OK: &#x201c;The following hostnames are not valid FQDNs&#x201d;</p>
+<p><b>Choose Services:</b> Select following Services: HDFS YARN + MapReduce2 Tez Hive HBase Pig Zookeeper Storm Flume Ambari Metrics Kafka Spark Zeppelin Notebook Elasticsearch Kibana Metron Slider</p>
+<p><b>Assign Masters:</b> Assign &#x201c;Kafka Broker&#x201d; on all nodes. Make sure move following components on one common node (Taken from previous guide, is this still necessary?): Storm UI Server Metron Indexing MySQL Server Kibana Server Elasticsearch Master Metron Parsers Metron Enrichment</p>
+<p><b>Assign Slaves and Clients:</b> select All for: DataNode NodeManager RegionServer Supervisor Client</p>
+<p><b>Customize Services:</b> Following is a list of services that need to be configured:</p>
+
+<ul>
+  
+<li>
+<p>Set the &#x201c;NameNode Java heap size&#x201d; (namenode_heapsize) from the default 1024 MB to at least 4096 MB under HDFS -&gt; Configs.</p></li>
+  
+<li>
+<p>For ElasticSearch:</p>
+  
+<ul>
+    
+<li>Set &#x201c;zen_discovery_ping_unicast_hosts&#x201d; to the IP of the node where you assigned ElasticSearch Master on the Assign Master tab.</li>
+    
+<li>Under &#x201c;Advanced elastic-site&#x201d;: Change &#x201c;network_host&#x201d; to &#x201c;0.0.0.0&#x201d;. Do not do this if your Metron is exposed to the public internet! Is &#x201c;[ <i>local</i>, <i>site</i> ]&#x201d; now.</li>
+  </ul></li>
+  
+<li>
+<p>Kibana:</p>
+  
+<ul>
+    
+<li>Set &#x201c;kibana_es_url&#x201d; to <tt>http://&lt;replace_with_elasticsearch_master_hostname&gt;:9200</tt>. &#x201c;replace_with_elasticsearch_master_hostname&#x201d; is the IP of the node where you assigned ElasticSearch Master on the Assign Master tab.</li>
+    
+<li>Change kibana_default_application to &#x201c;dashboard/Metron-Dashboard&#x201d;</li>
+  </ul></li>
+  
+<li>
+<p>Metron: Set &#x201c;Elasticsearch Hosts&#x201d; to the IP of the node where you assigned ElasticSearch Master on the Assign Master tab.</p></li>
+  
+<li>
+<p>Storm: You might have to increase the number of supervisor.slots.ports from the default &#x201c;[6700, 6701]&#x201d; to [6700, 6701, 6702, 6703, 6704] if you&#x2019;re only installing a single node.</p></li>
+  
+<li>
+<p>For metron REST use:</p>
+  
+<div class="source">
+<div class="source">
+<pre>Metron JDBC client path: /usr/share/java/mysql-connector-java.jar
+Metron JDBC Driver: com.mysql.jdbc.Driver
+Metron JDBC password: &lt;DB PASSWORD&gt;
+Metron JDBC platform: mysql
+Metron JDBC URL: jdbc:mysql://127.0.0.1:3306/&lt;DB NAME&gt;
+Metron JDBC username: &lt;DB USERNAME&gt;
+</pre></div></div></li>
+  
+<li>
+<p>Set rest of the configuration values to recommended by Ambari or the ones you desire (like DB passwords) and perform install. In a 3 node cluster, I ended up with:</p></li>
+</ul>
+
+<table border="0" class="table table-striped">
+  <thead>
+    
+<tr class="a">
+      
+<th>node1 </th>
+      
+<th>node2 </th>
+      
+<th>node3</th>
+    </tr>
+  </thead>
+  <tbody>
+    
+<tr class="b">
+      
+<td>DataNode </td>
+      
+<td>App Timeline Server </td>
+      
+<td>DataNode</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>Elasticsearch Master </td>
+      
+<td>DataNode </td>
+      
+<td>Elasticsearch Data Node</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>HBase Client </td>
+      
+<td>DRPC Server </td>
+      
+<td>Flume</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>HBase Master </td>
+      
+<td>HBase Client </td>
+      
+<td>HBase Client</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>RegionServer </td>
+      
+<td>RegionServer </td>
+      
+<td>RegionServer</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>HCat Client </td>
+      
+<td>HCat Client </td>
+      
+<td>HCat Client</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>HDFS Client </td>
+      
+<td>HDFS Client </td>
+      
+<td>HDFS Client</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>Hive Client </td>
+      
+<td>History Server </td>
+      
+<td>Hive Client</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>Kafka Broker </td>
+      
+<td>Hive Client </td>
+      
+<td>Kafka Broker</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>Kibana Server </td>
+      
+<td>Hive Metastore </td>
+      
+<td>MapReduce2 Client</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>MapReduce2 Client </td>
+      
+<td>HiveServer2 </td>
+      
+<td>Metrics Collector</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>Grafana </td>
+      
+<td>Kafka Broker </td>
+      
+<td>Metrics Monitor</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>Metrics Monitor </td>
+      
+<td>MapReduce2 Client </td>
+      
+<td>Metron Client</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>Metron Client </td>
+      
+<td>Metrics Monitor </td>
+      
+<td>NodeManager</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>Metron Enrichment </td>
+      
+<td>Metron Client </td>
+      
+<td>Pig Client</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>Metron Indexing </td>
+      
+<td>MySQL Server </td>
+      
+<td>Slider Client</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>Metron Parsers </td>
+      
+<td>Nimbus </td>
+      
+<td>Spark Client</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>Metron REST </td>
+      
+<td>NodeManager </td>
+      
+<td>Supervisor</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>NameNode </td>
+      
+<td>Pig Client </td>
+      
+<td>Tez Client</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>NodeManager </td>
+      
+<td>ResourceManager </td>
+      
+<td>YARN Client</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>Pig Client </td>
+      
+<td>SNameNode </td>
+      
+<td>ZooKeeper Client</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>Slider Client </td>
+      
+<td>Slider Client </td>
+      
+<td>ZooKeeper Server</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>Spark Client </td>
+      
+<td>Spark Client </td>
+    </tr>
+    
+<tr class="a">
+      
+<td>Spark History Server </td>
+      
+<td>Supervisor </td>
+    </tr>
+    
+<tr class="b">
+      
+<td>Storm UI Server </td>
+      
+<td>Tez Client</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>Supervisor </td>
+      
+<td>WebHCat Server</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>Tez Client </td>
+      
+<td>YARN Client</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>YARN Client </td>
+      
+<td>ZooKeeper Client</td>
+    </tr>
+    
+<tr class="b">
+      
+<td>Zeppelin Notebook </td>
+      
+<td>ZooKeeper Server</td>
+    </tr>
+    
+<tr class="a">
+      
+<td>ZooKeeper Client </td>
+    </tr>
+    
+<tr class="b">
+      
+<td>ZooKeeper Server </td>
+    </tr>
+  </tbody>
+</table>
+
+<ul>
+  
+<li>
+<p>Install everything. Metron REST will probably not work as we still need to add a user and the database to MySQL.</p></li>
+  
+<li>
+<p>Configure a user for Metron REST in MySQL. On the node where you installed the Metron REST UI, do:</p></li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># mysql -u root -p
+CREATE USER '&lt;DB USERNAME&gt;'@'localhost' IDENTIFIED BY '&lt;DB PASSWORD&gt;';
+CREATE DATABASE IF NOT EXISTS &lt;DB NAME&gt;;
+GRANT ALL PRIVILEGES ON &lt;DB NAME&gt;.* TO '&lt;DB USERNAME&gt;'@'localhost';
+</pre></div></div>
+<p>For example:</p>
+
+<div class="source">
+<div class="source">
+<pre># mysql -u root -p
+&gt; CREATE USER 'metron'@'localhost' IDENTIFIED BY 'metron';
+&gt; CREATE DATABASE IF NOT EXISTS metronrest;
+&gt; GRANT ALL PRIVILEGES ON metronrest.* TO 'metron'@'localhost';
+&gt; quit
+Bye
+# 
+</pre></div></div>
+<p>Add the Metron REST username and password to the metronrest database:</p>
+
+<div class="source">
+<div class="source">
+<pre># mysql -u &lt;DB USERNAME&gt; -p
+&gt; use &lt;DB NAME&gt;;
+&gt; insert into users (username, password, enabled) values ('&lt;USERNAME&gt;','&lt;PASSWORD&gt;',1);
+&gt; insert into authorities (username, authority) values ('&lt;USERNAME&gt;', 'ROLE_USER');
+&gt; quit
+Bye
+#
+</pre></div></div>
+<p>For example, to use the username &#x2018;metron&#x2019; with password &#x2018;metron&#x2019;, do the following:</p>
+
+<div class="source">
+<div class="source">
+<pre># mysql -u metron -p
+&gt; use metronrest;
+&gt; insert into users (username, password, enabled) values ('metron','metron',1);
+&gt; insert into authorities (username, authority) values ('metron', 'ROLE_USER');
+&gt; quit
+Bye
+#
+</pre></div></div>
+<p>Make sure that all the services are up.</p>
+<p>Install metron_pcapservice:</p>
+
+<div class="source">
+<div class="source">
+<pre># cp /root/metron/metron-platform/metron-api/target/metron-api-0.4.0.jar /usr/metron/0.4.0/lib/
+# wget -O /etc/init.d/pcapservice https://raw.githubusercontent.com/apache/metron/master/metron-deployment/roles/metron_pcapservice/templates/pcapservice
+# sed -i 's/{{ pcapservice_jar_dst }}/\/usr\/metron\/0.4.0\/lib\/metron-api-0.4.0.jar/' /etc/init.d/pcapservice
+# sed -i 's/{{ pcapservice_port }}/8081/' /etc/init.d/pcapservice
+# sed -i 's/{{ query_hdfs_path }}/\/tmp/' /etc/init.d/pcapservice
+# sed -i 's/{{ pcap_hdfs_path }}/\/apps\/metron\/pcap/' /etc/init.d/pcapservice
+# chmod 755 /etc/init.d/pcapservice
+# wget -O /etc/logrotate.d/metron-pcapservice https://raw.githubusercontent.com/apache/metron/master/metron-deployment/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml
+# sed -i 's/^  {{ metron_pcapservice_logrotate_frequency }}.*$/  daily/' /etc/logrotate.d/metron-pcapservice
+# sed -i 's/^  rotate {{ metron_pcapservice_logrotate_retention }}.*$/  rotate 30/' /etc/logrotate.d/metron-pcapservice
+# chmod 644 /etc/logrotate.d/metron-pcapservice
+</pre></div></div>
+<p>Install tap interface:</p>
+
+<div class="source">
+<div class="source">
+<pre># yum install tunctl -y
+# tunctl -p
+</pre></div></div>
+<p>Bring up tap0 on 10.0.0.100:</p>
+
+<div class="source">
+<div class="source">
+<pre># ifconfig tap0 10.0.0.100 up
+# ip link set tap0 promisc on
+</pre></div></div>
+<p>Install librdkafka:</p>
+
+<div class="source">
+<div class="source">
+<pre># yum install cmake make gcc gcc-c++ flex bison libpcap libpcap-devel openssl-devel python-devel swig zlib-devel perlcyrus-sasl cyrus-sasl-devel cyrus-sasl-gssapi -y
+# cd /tmp
+# wget -O /tmp/librdkafka-0.9.4.tar.gz https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz
+# /bin/gtar --extract -C /tmp -z -f /tmp/librdkafka-0.9.4.tar.gz
+# cd /tmp/librdkafka-0.9.4
+# ./configure --prefix=/usr/local --enable-sasl
+# make
+# make install
+</pre></div></div>
+<p>Install pycapa</p>
+
+<div class="source">
+<div class="source">
+<pre># yum install centos-release-scl -y
+# yum update -y
+# yum install python27 -y
+# scl enable python27 bash
+
+# cd /opt/rh/python27/root/usr/bin/
+# LD_LIBRARY_PATH=$LD_LIBRARY_PATH ./pip2.7 install --upgrade pip
+# LD_LIBRARY_PATH=$LD_LIBRARY_PATH ./pip2.7 install requests
+
+
+(# /opt/rh/python27/root/usr/bin/virtualenv py27venv
+# source py27venv/bin/activate
+# pip install --upgrade pip
+# pip install ansible==2.0.0.2
+# ansible --version
+# deactivate)
+</pre></div></div>
+
+<div class="source">
+<div class="source">
+<pre># yum install @Development python-virtualenv libpcap-devel libselinux-python -y
+# mkdir /usr/local/pycapa
+# cd /usr/local/pycapa
+# virtualenv pycapa-venv
+# source pycapa-venv/bin/activate
+# cp -r /root/metron/metron-sensors/pycapa/. /usr/local/pycapa/.
+# pip install --upgrade pip
+# /usr/local/pycapa/pycapa-venv/bin/pip install -r requirements.txt
+(# pip install -r requirements.txt)
+
+# /usr/local/pycapa/pycapa-venv/bin/python setup.py install
+# ln -s /usr/local/lib/librdkafka.so.1 /opt/rh/python27/root/usr/lib64
+# deactivate
+</pre></div></div>
+<p>Log out and log in to make sure Python is back to version 2.6 instead of 2.7.</p>
+
+<div class="source">
+<div class="source">
+<pre># wget -O /etc/init.d/pycapa https://raw.githubusercontent.com/apache/metron/master/metron-deployment/roles/pycapa/templates/pycapa
+# sed -i 's/{{ pycapa_log }}/\/var\/log\/pycapa.log/' /etc/init.d/pycapa
+# sed -i 's/{{ pycapa_home }}/\/usr\/local\/pycapa/' /etc/init.d/pycapa
+# sed -i 's/{{ python27_home }}/\/opt\/rh\/python27\/root/' /etc/init.d/pycapa
+# sed -i 's/{{ pycapa_bin }}/\/usr\/local\/pycapa\/pycapa-venv\/bin/' /etc/init.d/pycapa
+# sed -i 's/--kafka {{ kafka_broker_url }}/--kafka-broker &lt;IP:6667&gt;/' /etc/init.d/pycapa
+# sed -i 's/--topic {{ pycapa_topic }}/--kafka-topic pcap/' /etc/init.d/pycapa
+# sed -i 's/{{ pycapa_sniff_interface }}/tap0/' /etc/init.d/pycapa
+(# sed -i 's/export LD_LIBRARY_PATH=\/opt\/rh\/python27\/root\/usr\/lib64/export LD_LIBRARY_PATH=\/usr\/local\/lib/' /etc/init.d/pycapa)
+# chmod 755 /etc/init.d/pycapa
+# yum install @Development libdnet-devel rpm-build libpcap libpcap-devel pcre pcre-devel zlib zlib-devel glib2-devel -y
+# yum install kafka -y
+</pre></div></div>
+<p>Install bro:</p>
+
+<div class="source">
+<div class="source">
+<pre># wget -O /tmp/bro-2.4.1.tar.gz https://www.bro.org/downloads/release/bro-2.4.1.tar.gz
+# /bin/gtar --extract -C /tmp -z -f /tmp/bro-2.4.1.tar.gz
+# cd /tmp/bro-2.4.1
+# ./configure --prefix=/usr/local/bro
+# make -j4
+# make install
+</pre></div></div>
+<p>Configure bro:</p>
+
+<div class="source">
+<div class="source">
+<pre># sed -i 's/interface=eth0/interface=tap0/' /usr/local/bro/etc/node.cfg
+# /usr/local/bro/bin/broctl install
+</pre></div></div>
+<p>Edit crontab with <tt># crontab -e</tt> and add:</p>
+
+<div class="source">
+<div class="source">
+<pre>0-59/5  *   *   *   *   /usr/local/bro/bin/broctl cron
+0-59/5  *   *   *   *   rm -rf /usr/local/bro/spool/tmp/*
+</pre></div></div>
+<p>bro-kafka:</p>
+
+<div class="source">
+<div class="source">
+<pre># cp -r /root/metron/metron-sensors/bro-plugin-kafka /tmp
+# cd /tmp/bro-plugin-kafka
+# rm -rf build/
+# ./configure --bro-dist=/tmp/bro-2.4.1 --install-root=/usr/local/bro/lib/bro/plugins/ --with-librdkafka=/usr/local
+# make -j4
+# make install
+</pre></div></div>
+<p>Configure bro-kafka plugin:</p>
+
+<div class="source">
+<div class="source">
+<pre># cat &lt;&lt; EOF &gt;&gt; /usr/local/bro/share/bro/site/local.bro
+@load Bro/Kafka/logs-to-kafka.bro
+redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG);
+redef Kafka::topic_name = &quot;bro&quot;;
+redef Kafka::tag_json = T;
+redef Kafka::kafka_conf = table( [&quot;metadata.broker.list&quot;] = &quot;&lt;KAFKA_BROKER_IP&gt;:6667&quot; );
+EOF
+# /usr/local/bro/bin/broctl deploy
+# ip link set tap0 promisc on
+</pre></div></div>
+<p>Install daq:</p>
+
+<div class="source">
+<div class="source">
+<pre># wget -O /tmp/daq-2.0.6-1.src.rpm https://snort.org/downloads/snort/daq-2.0.6-1.src.rpm
+# cd /tmp
+# rpmbuild --rebuild daq-2.0.6-1.src.rpm
+</pre></div></div>
+<p>This last command creates the files /root/rpmbuild/RPMS/x86_64/daq-2.0.6-1.x86_64.rpm &amp; /root/rpmbuild/RPMS/x86_64/daq-debuginfo-2.0.6-1.x86_64.rpm. We only need to install the first rpm.</p>
+
+<div class="source">
+<div class="source">
+<pre># yum install /root/rpmbuild/RPMS/x86_64/daq-2.0.6-1.x86_64.rpm -y
+</pre></div></div>
+<p>Install snort:</p>
+
+<div class="source">
+<div class="source">
+<pre># wget -O /tmp/snort-2.9.8.0-1.src.rpm https://snort.org/downloads/archive/snort/snort-2.9.8.0-1.src.rpm
+# cd /tmp
+# rpmbuild --rebuild snort-2.9.8.0-1.src.rpm
+</pre></div></div>
+<p>This last command creates the files /root/rpmbuild/RPMS/x86_64/snort-2.9.8.0-1.x86_64.rpm &amp; /root/rpmbuild/RPMS/x86_64/snort-debuginfo-2.9.8.0-1.x86_64.rpm. We only need to install the first rpm.</p>
+
+<div class="source">
+<div class="source">
+<pre># yum install /root/rpmbuild/RPMS/x86_64/snort-2.9.8.0-1.x86_64.rpm -y
+# wget -O /tmp/community-rules.tar.gz https://www.snort.org/downloads/community/community-rules.tar.gz
+# /bin/gtar --extract -C /tmp -z -f /tmp/community-rules.tar.gz
+# cp -r community-rules/community.rules /etc/snort/rules
+# touch /etc/snort/rules/white_list.rules
+# touch /etc/snort/rules/black_list.rules
+# touch /var/log/snort/alerts
+# chown -R snort:snort /etc/snort
+# sed -i 's/^# alert/alert/' /etc/snort/rules/community.rules
+# wget -O /tmp/snort.conf https://github.com/apache/metron/raw/master/metron-deployment/roles/snort/files/snort.conf
+# cp snort.conf /etc/snort/snort.conf
+# sed -i 's/^ipvar HOME_NET.*$/ipvar HOME_NET any/' /etc/snort/snort.conf
+# echo &quot;output alert_csv: /var/log/snort/alert.csv default&quot; &gt;&gt; /etc/snort/snort.conf
+# sed -i 's/^ALERTMODE=.*$/ALERTMODE=/' /etc/sysconfig/snort
+# sed -i 's/^NO_PACKET_LOG=.*$/NO_PACKET_LOG=1/' /etc/sysconfig/snort
+# sed -i 's/^INTERFACE=.*$/INTERFACE=tap0/' /etc/sysconfig/snort
+# mkdir /opt/snort-producer
+# chmod 755 /opt/snort-producer
+</pre></div></div>
+
+<div class="source">
+<div class="source">
+<pre># wget -O /opt/snort-producer/start-snort-producer.sh https://github.com/apache/metron/raw/master/metron-deployment/roles/snort/templates/start-snort-producer.sh
+# sed -i 's/{{ snort_alert_csv_path }}/\/var\/log\/snort\/alert.csv/' /opt/snort-producer/start-snort-producer.sh
+# sed -i 's/{{ kafka_prod }}/\/usr\/hdp\/current\/kafka-broker\/bin\/kafka-console-producer.sh/' /opt/snort-producer/start-snort-producer.sh
+# sed -i 's/{{ kafka_broker_url }}/&lt;KAFKA_BROKER_IP&gt;:6667/' /opt/snort-producer/start-snort-producer.sh
+# sed -i 's/{{ snort_topic }}/snort/' /opt/snort-producer/start-snort-producer.sh
+# chmod 755 /opt/snort-producer/start-snort-producer.sh
+</pre></div></div>
+
+<div class="source">
+<div class="source">
+<pre># wget -O /etc/init.d/snort-producer https://github.com/apache/metron/raw/master/metron-deployment/roles/snort/templates/snort-producer
+# sed -i 's/{{ snort_producer_home }}/\/opt\/snort-producer/' /etc/init.d/snort-producer
+# sed -i 's/{{ snort_producer_start }}/\/opt\/snort-producer\/start-snort-producer.sh/' /etc/init.d/snort-producer
+# chmod 755 /etc/init.d/snort-producer
+</pre></div></div>
+<p>Install yaf:</p>
+
+<div class="source">
+<div class="source">
+<pre># wget -O /tmp/libfixbuf-1.7.1.tar.gz http://tools.netsa.cert.org/releases/libfixbuf-1.7.1.tar.gz
+# /bin/gtar --extract -C /tmp -z -f /tmp/libfixbuf-1.7.1.tar.gz
+# cd /tmp/libfixbuf-1.7.1
+# ./configure
+# make -j4
+# make install
+# wget -O /tmp/yaf-2.8.0.tar.gz http://tools.netsa.cert.org/releases/yaf-2.8.0.tar.gz
+# /bin/gtar --extract -C /tmp -z -f /tmp/yaf-2.8.0.tar.gz
+# cd /tmp/yaf-2.8.0
+# ./configure --enable-applabel --enable-plugins
+# make -j4
+# make install
+# mkdir /opt/yaf
+# chmod 755 /opt/yaf
+# wget -O /opt/yaf/start-yaf.sh https://github.com/apache/metron/raw/master/metron-deployment/roles/yaf/templates/start-yaf.sh
+# sed -i 's/{{ yaf_bin }}/\/usr\/local\/bin\/yaf/' /opt/yaf/start-yaf.sh
+# sed -i 's/{{ sniff_interface }}/tap0/' /opt/yaf/start-yaf.sh
+# sed -i 's/{{ yafscii_bin }}/\/usr\/local\/bin\/yafscii/' /opt/yaf/start-yaf.sh
+# sed -i 's/{{ kafka_prod }}/\/usr\/hdp\/current\/kafka-broker\/bin\/kafka-console-producer.sh/' /opt/yaf/start-yaf.sh
+# sed -i 's/{{ kafka_broker_url }}/&lt;BROKER_IP&gt;:6667/' /opt/yaf/start-yaf.sh
+# sed -i 's/{{ yaf_topic }}/yaf/' /opt/yaf/start-yaf.sh
+# chmod 755 /opt/yaf/start-yaf.sh
+# wget -O /etc/init.d/yaf https://github.com/apache/metron/raw/master/metron-deployment/roles/yaf/templates/yaf
+# sed -i 's/{{ yaf_home }}/\/opt\/yaf/' /etc/init.d/yaf
+# sed -i 's/{{ yaf_start }}/\/opt\/yaf\/start-yaf.sh/' /etc/init.d/yaf
+# sed -i 's/^DAEMONOPTS=\&quot;${@:2}\&quot;$/DAEMONOPTS=\&quot;${@:2} --idle-timeout 0\&quot;/' /etc/init.d/yaf
+# chmod 755 /etc/init.d/yaf
+</pre></div></div>
+<p>Install tcpreplay:</p>
+
+<div class="source">
+<div class="source">
+<pre># wget -O /tmp/tcpreplay-4.1.1.tar.gz https://github.com/appneta/tcpreplay/releases/download/v4.1.1/tcpreplay-4.1.1.tar.gz
+# /bin/gtar --extract -C /opt -z  -f /tmp/tcpreplay-4.1.1.tar.gz
+# cd /opt/tcpreplay-4.1.1/
+# ./configure --prefix=/opt
+# make -j4
+# make install
+# mkdir /opt/pcap-replay
+# chown root.root /opt/pcap-replay
+# chmod 755 /opt/pcap-replay
+# cd /opt/pcap-replay
+# wget https://github.com/apache/metron/raw/master/metron-deployment/roles/sensor-test-mode/files/example.pcap
+# echo &quot;include \$RULE_PATH/test.rules&quot; &gt;&gt; /etc/snort/snort.conf
+# echo &quot;alert tcp any any -&gt; any any (msg:'snort test alert'; sid:999158; )&quot; &gt; /etc/snort/rules/test.rules
+# wget -O /etc/init.d/pcap-replay https://github.com/apache/metron/raw/master/metron-deployment/roles/pcap_replay/templates/pcap-replay
+# sed -i 's/{{ pcap_replay_home }}/\/opt\/pcap-replay/' /etc/init.d/pcap-replay
+# sed -i 's/{{ pcap_replay_interface }}/tap0/' /etc/init.d/pcap-replay
+# sed -i 's/{{ tcpreplay_prefix }}/\/opt/' /etc/init.d/pcap-replay
+# chmod 755 /etc/init.d/pcap-replay
+</pre></div></div>
+<p>Install monit</p>
+
+<div class="source">
+<div class="source">
+<pre># yum install monit -y
+# wget -O /etc/monit.conf https://github.com/apache/metron/raw/master/metron-deployment/roles/monit/templates/monit/monit.conf
+
+# sed -i 's/{{ inventory_hostname }}/&lt;IP ADDRESS&gt;/' /etc/monit.conf
+# sed -i 's/{{ monit_user }}/admin/' /etc/monit.conf
+# sed -i 's/{{ monit_pass }}/monit/' /etc/monit.conf
+# chmod 600 /etc/monit.conf
+
+# wget -O /etc/monit.d/pcap-replay.monit https://github.com/apache/metron/raw/master/metron-deployment/roles/monit/templates/monit/pcap-replay.monit
+# chmod 644 /etc/monit.d/pcap-replay.monit
+
+# wget -O /etc/monit.d/pcap-service.monit https://github.com/apache/metron/raw/master/metron-deployment/roles/monit/templates/monit/pcap-service.monit
+# chmod 644 /etc/monit.d/pcap-service.monit
+
+# wget -O /etc/monit.d/pycapa.monit https://github.com/apache/metron/raw/master/metron-deployment/roles/monit/templates/monit/pycapa.monit
+# chmod 644 /etc/monit.d/pycapa.monit
+
+# wget -O /etc/monit.d/snort.monit https://github.com/apache/metron/raw/master/metron-deployment/roles/monit/templates/monit/snort.monit
+# chmod 644 /etc/monit.d/snort.monit
+
+# wget -O /etc/monit.d/yaf.monit https://github.com/apache/metron/raw/master/metron-deployment/roles/monit/templates/monit/yaf.monit
+# chmod 644 /etc/monit.d/yaf.monit
+
+# wget -O /etc/monit.d/bro.monit https://github.com/apache/metron/raw/master/metron-deployment/roles/monit/templates/monit/bro.monit
+# sed -i 's/^  with pidfile.*$/  with pidfile \/usr\/local\/bro\/spool\/bro\/\.pid/' /etc/monit.d/bro.monit
+# chmod 644 /etc/monit.d/bro.monit
+
+# service monit start
+# chkconfig --list monit
+# chkconfig monit on
+# chkconfig --list monit
+# monit reload
+# monit stop all
+# monit start all
+# monit summary | tail -n +3 | awk -F&quot;'&quot; '{print $2}'
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Miscellaneous_Issues"></a>Miscellaneous Issues</h3>
+
+<ul>
+  
+<li>I had a problem with Zeppelin after rebooting this machine and had to manually create the Zeppelin run directory:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># mkdir /var/run/zeppelin
+# chown zeppelin.hadoop zeppelin/
+</pre></div></div>
+
+<ul>
+  
+<li>Additionally, while working with Metron, I&#x2019;ve noticed that at some point Zeppelin Notebook started, but immediately stopped again. In the logs, I could see &#x201c;Address already in use&#x201d; messages. It turns out that there was still a lingering Zeppelin process on the host. To fix it, stop Zeppelin Notebook in Ambari and then kill the latent process:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># ps aux | grep zeppelin
+# kill &lt;zeppelin_java_pid&gt;
+</pre></div></div>
+<p>Afterwards, restart Zeppelin Notebook via Ambari.</p>
+
+<ul>
+  
+<li>I had a couple of issues with Elasticsearch where it wouldn&#x2019;t find a master. This was fixed by doing the following. In Ambari, set the following items: &#x201c;masters_also_are_datanodes&#x201d; to &#x201c;true&#x201d; &#x201c;expected_data_nodes&#x201d; = &#x201c;0&#x201d; &#x201c;gateway_recover_after_data_nodes&#x201d; = &#x201c;1&#x201d; Restart all services. At this point, I noticed the following in :/etc/elasticsearch/elasticsearch.yml&quot;:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre>node:
+  data: true
+  master: true
+  name: metron1.local
+</pre></div></div>
+<p>After changing this to :</p>
+
+<div class="source">
+<div class="source">
+<pre>node:
+  data: true
+  master: true
+  name: metron
+</pre></div></div>
+<p>and restarting elasticsearch with &#x201c;service elasticsearch restart&#x201d;, elasticsearch started indexing.</p>
+
+<ul>
+  
+<li>Another with Elasticsearch was that I saw the following error message in Kibana:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre>plugin:elasticsearch     Elasticsearch is still initializing the kibana index.
+</pre></div></div>
+<p>This was fixed by deleting the Kibana index &#x201c;.kibana&#x201d;: <tt>curl -XDELETE http://localhost:9200/.kibana</tt></p></div>
+<div class="section">
+<h3><a name="Miscellaneous_Services"></a>Miscellaneous Services</h3>
+
+<ul>
+  
+<li>Load the correct Elasticsearch template with:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre>curl -s -w &quot;%{http_code}&quot; -u &lt;USERNAME&gt;:&lt;PASSWORD&gt; -H &quot;X-Requested-By: ambari&quot; -X POST -d '{ &quot;RequestInfo&quot;: { &quot;context&quot;: &quot;Install ES Template from REST&quot;, &quot;command&quot;: &quot;ELASTICSEARCH_TEMPLATE_INSTALL&quot;},&quot;Requests/resource_filters&quot;: [{&quot;service_name&quot;: &quot;METRON&quot;,&quot;component_name&quot;: &quot;METRON_INDEXING&quot;,&quot;hosts&quot; : &quot;&lt;HOSTNAME&gt;&quot;}]}' http://&lt;AMBARI HOST&gt;:8080/api/v1/clusters/&lt;CLUSTERNAME&gt;/requests
+</pre></div></div>
+<p>For example:</p>
+
+<div class="source">
+<div class="source">
+<pre>curl -s -w &quot;%{http_code}&quot; -u admin:admin -H &quot;X-Requested-By: ambari&quot; -X POST -d '{ &quot;RequestInfo&quot;: { &quot;context&quot;: &quot;Install ES Template from REST&quot;, &quot;command&quot;: &quot;ELASTICSEARCH_TEMPLATE_INSTALL&quot;},&quot;Requests/resource_filters&quot;: [{&quot;service_name&quot;: &quot;METRON&quot;,&quot;component_name&quot;: &quot;METRON_INDEXING&quot;,&quot;hosts&quot; : &quot;metron&quot;}]}' http://192.168.10.10:8080/api/v1/clusters/metron/requests
+</pre></div></div>
+
+<ul>
+  
+<li>Load Kibana Dashboard with:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre>curl -s -w &quot;%{http_code}&quot; -u &lt;USERNAME&gt;:&lt;PASSWORD&gt; -H &quot;X-Requested-By: ambari&quot; -X POST -d '{ &quot;RequestInfo&quot;: { &quot;context&quot;: &quot;Install Kibana Dashboard from REST&quot;, &quot;command&quot;: &quot;LOAD_TEMPLATE&quot;},&quot;Requests/resource_filters&quot;: [{&quot;service_name&quot;: &quot;KIBANA&quot;,&quot;component_name&quot;: &quot;KIBANA_MASTER&quot;,&quot;hosts&quot; : &quot;&lt;HOSTNAME&gt;&quot;}]}' http://&lt;AMBARI HOST&gt;:8080/api/v1/clusters/&lt;CLUSTERNAME&gt;/requests
+</pre></div></div>
+<p>For example:</p>
+
+<div class="source">
+<div class="source">
+<pre>curl -s -w &quot;%{http_code}&quot; -u admin:admin -H &quot;X-Requested-By: ambari&quot; -X POST -d '{ &quot;RequestInfo&quot;: { &quot;context&quot;: &quot;Install Kibana Dashboard from REST&quot;, &quot;command&quot;: &quot;LOAD_TEMPLATE&quot;},&quot;Requests/resource_filters&quot;: [{&quot;service_name&quot;: &quot;KIBANA&quot;,&quot;component_name&quot;: &quot;KIBANA_MASTER&quot;,&quot;hosts&quot; : &quot;metron&quot;}]}' http://192.168.10.10:8080/api/v1/clusters/metron/requests
+</pre></div></div>
+
+<ul>
+  
+<li>If you installed Metron on a single node, you might have to increase the number of Storm supervisor slots from the default 2 to 5 or more. This can be done by editing the &#x201c;supervisor.slots.ports&#x201d; under Storm in the Ambari UI. Change:</li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre>supervisor.slots.ports: [6700, 6701]
+</pre></div></div>
+<p>To:</p>
+
+<div class="source">
+<div class="source">
+<pre>supervisor.slots.ports: [6700, 6701, 6702, 6703, 6704, 6705]
+</pre></div></div>
+
+<ul>
+  
+<li>Install Apache NiFi. Download nifi-1.2.0-bin.tar.gz from <a class="externalLink" href="https://nifi.apache.org/download.html">https://nifi.apache.org/download.html</a></li>
+</ul>
+
+<div class="source">
+<div class="source">
+<pre># wget http://apache.mirror.iweb.ca/nifi/1.2.0/nifi-1.2.0-bin.tar.gz
+# tar xf nifi-1.2.0-bin.tar.gz
+</pre></div></div>
+<p>Before we run NiFi, we need to change the port as the default port collides with the Ambari port. To do this, we need to change the value &#x201c;nifi.web.http.port=8080&#x201d; to &#x201c;nifi.web.http.port=8089&#x201d; in the file &#x201c;nifi-1.1.2/conf/nifi.properties&#x201d;. Install and start NiFi afterwards:</p>
+
+<div class="source">
+<div class="source">
+<pre># nifi-1.2.0/bin/nifi.sh install
+# nifi-1.2.0/bin/nifi.sh start
+</pre></div></div></div>
+<div class="section">
+<h3><a name="Exposed_Interfaces"></a>Exposed Interfaces</h3>
+<p>In the end, you&#x2019;ll end up with a bunch of exposed UIs:</p>
+
+<ul>
+  
+<li>Ambari: <a class="externalLink" href="http://node1:8080/">http://node1:8080/</a></li>
+  
+<li>Kibana: <a class="externalLink" href="http://node1:5000/">http://node1:5000/</a></li>
+  
+<li>Sensor Status (monit): <a class="externalLink" href="http://node1:2812">http://node1:2812</a></li>
+  
+<li>Elasticsearch: <a class="externalLink" href="http://node1:9200/">http://node1:9200/</a></li>
+  
+<li>Storm UI: <a class="externalLink" href="http://node1:8744/">http://node1:8744/</a></li>
+  
+<li>Metron REST interface: <a class="externalLink" href="http://node1:8082/swagger-ui.html#/">http://node1:8082/swagger-ui.html#/</a></li>
+  
+<li>Management UI: <a class="externalLink" href="http://node1:4200/">http://node1:4200/</a> (user/password)</li>
+  
+<li>Apache Nifi: <a class="externalLink" href="http://node1:8089/nifi/">http://node1:8089/nifi/</a></li>
+  
+<li>Zookeeper: <a class="externalLink" href="http://node1:2181">http://node1:2181</a></li>
+  
+<li>Kafka: <a class="externalLink" href="http://node1:6667">http://node1:6667</a></li>
+</ul></div>
+<div class="section">
+<h3><a name="TROUBLESHOOTING"></a>TROUBLESHOOTING</h3></div></div>
+                  </div>
+            </div>
+          </div>
+
+    <hr/>
+
+    <footer>
+            <div class="container-fluid">
+              <div class="row span12">Copyright &copy;                    2017
+                        <a href="https://www.apache.org">The Apache Software Foundation</a>.
+            All Rights Reserved.      
+                    
+      </div>
+
+                          
+        
+                </div>
+    </footer>
+  </body>
+</html>


Mime
View raw message