eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h..@apache.org
Subject svn commit: r1789966 [4/4] - in /eagle/site/docs: latest/ latest/applications/ latest/css/ latest/developing-application/ latest/getting-started/ latest/include/images/ latest/js/ latest/reference/ latest/underlying-design/ latest/using-eagle/ v0.5.0/c...
Date Mon, 03 Apr 2017 11:35:51 GMT
Added: eagle/site/docs/latest/using-eagle/index.html
URL: http://svn.apache.org/viewvc/eagle/site/docs/latest/using-eagle/index.html?rev=1789966&view=auto
==============================================================================
--- eagle/site/docs/latest/using-eagle/index.html (added)
+++ eagle/site/docs/latest/using-eagle/index.html Mon Apr  3 11:35:50 2017
@@ -0,0 +1,1178 @@
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  
+  
+  <title>Using Eagle - Apache Eagle Documentation</title>
+  
+
+  <link rel="shortcut icon" href="../include/images/favicon.png">
+  
+
+  
+  <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
+
+  <link rel="stylesheet" href="../css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../css/theme_extra.css" type="text/css" />
+  <link rel="stylesheet" href="../css/highlight.css">
+
+  
+  <script>
+    // Current page data
+    var mkdocs_page_name = "Using Eagle";
+    var mkdocs_page_input_path = "using-eagle.md";
+    var mkdocs_page_url = "/using-eagle/";
+  </script>
+  
+  <script src="../js/jquery-2.1.1.min.js"></script>
+  <script src="../js/modernizr-2.8.3.min.js"></script>
+  <script type="text/javascript" src="../js/highlight.pack.js"></script>
+  <script src="../js/theme.js"></script> 
+
+  
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+      <div class="wy-side-nav-search">
+        <a href=".." class="icon icon-home"> Apache Eagle Documentation</a>
+        <div role="search">
+  <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+  </form>
+</div>
+      </div>
+
+      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+        <ul class="current">
+          
+            <li><script>
+    function select(elem) {
+        var selectedClassName = "selected";
+        $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+        $(elem).addClass(selectedClassName);
+    }
+</script>
+
+<!-- 
+    this macro is to fetch the first child element that has url and return the url as the default one of the section
+    arguemnt:
+      ni: nav_item, corresponding to navigation item configured in mkdocs.yml's "pages"
+ -->
+
+
+<!-- 
+    this macro generates the style for indentation on sub tocs in side nav
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- 
+    this macro shows toc items including nested tocs, toc nesting level would depending on arguments described below
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+    <li class="toctree-l1 ">
+        <a class="" href="..">Home</a>
+        
+    </li>
+<li>
+          
+            <li><script>
+    function select(elem) {
+        var selectedClassName = "selected";
+        $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+        $(elem).addClass(selectedClassName);
+    }
+</script>
+
+<!-- 
+    this macro is to fetch the first child element that has url and return the url as the default one of the section
+    arguemnt:
+      ni: nav_item, corresponding to navigation item configured in mkdocs.yml's "pages"
+ -->
+
+
+<!-- 
+    this macro generates the style for indentation on sub tocs in side nav
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- 
+    this macro shows toc items including nested tocs, toc nesting level would depending on arguments described below
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+    <li class="toctree-l1 ">
+        <a class="" href="../getting-started/">Getting Started</a>
+        
+    </li>
+<li>
+          
+            <li><script>
+    function select(elem) {
+        var selectedClassName = "selected";
+        $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+        $(elem).addClass(selectedClassName);
+    }
+</script>
+
+<!-- 
+    this macro is to fetch the first child element that has url and return the url as the default one of the section
+    arguemnt:
+      ni: nav_item, corresponding to navigation item configured in mkdocs.yml's "pages"
+ -->
+
+
+<!-- 
+    this macro generates the style for indentation on sub tocs in side nav
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- 
+    this macro shows toc items including nested tocs, toc nesting level would depending on arguments described below
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+    <li class="toctree-l1 current">
+        <a class="current" href="./">Using Eagle</a>
+        
+            <ul>
+            
+                
+    
+        <li class="toctree-l3" onclick="select(this)"><a 
+    
+        
+    
+ href="#manage-eagle-and-services">Manage Eagle and Services</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l3" onclick="select(this)"><a 
+    
+        
+    
+ href="#use-eagle-web-interface">Use Eagle Web Interface</a></li>
+        
+            <ul>
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 4em;"
+    
+ href="#home">Home</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 4em;"
+    
+ href="#alert">Alert</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 4em;"
+    
+ href="#integration">Integration</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 4em;"
+    
+ href="#sites">Sites</a></li>
+        
+    
+
+            
+            </ul>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l3" onclick="select(this)"><a 
+    
+        
+    
+ href="#setup-the-monitoring-application">Setup The Monitoring Application</a></li>
+        
+            <ul>
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 4em;"
+    
+ href="#monitoring-applications">Monitoring Applications</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 4em;"
+    
+ href="#managing-sites">Managing Sites</a></li>
+        
+            <ul>
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 5em;"
+    
+ href="#sites_1">Sites</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 5em;"
+    
+ href="#create-site">Create Site</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 5em;"
+    
+ href="#configuring-a-site">Configuring a Site</a></li>
+        
+    
+
+            
+            </ul>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 4em;"
+    
+ href="#install-and-run-applications-in-site">Install and Run Applications in Site</a></li>
+        
+    
+
+            
+            </ul>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l3" onclick="select(this)"><a 
+    
+        
+    
+ href="#define-policies">Define Policies</a></li>
+        
+            <ul>
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 4em;"
+    
+ href="#policies">Policies</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 4em;"
+    
+ href="#define-or-edit-policies">Define or Edit Policies</a></li>
+        
+            <ul>
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 5em;"
+    
+ href="#source-stream">Source Stream</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 5em;"
+    
+ href="#policy-name">Policy Name</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 5em;"
+    
+ href="#publish-alerts">Publish Alerts</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 5em;"
+    
+ href="#policy-syntax">Policy Syntax</a></li>
+        
+    
+
+            
+            </ul>
+        
+    
+
+            
+            </ul>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l3" onclick="select(this)"><a 
+    
+        
+    
+ href="#monitoring-dashboard">Monitoring Dashboard</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l3" onclick="select(this)"><a 
+    
+        
+    
+ href="#check-the-alerts">Check The Alerts</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l3" onclick="select(this)"><a 
+    
+        
+    
+ href="#how-to-stream-audit-log-into-kafka">How to stream audit log into Kafka</a></li>
+        
+            <ul>
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 4em;"
+    
+ href="#logstash">Logstash</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 4em;"
+    
+ href="#filebeat">Filebeat</a></li>
+        
+    
+
+            
+                
+    
+        <li class="toctree-l4" onclick="select(this)"><a 
+    
+        
+        style="padding-left: 4em;"
+    
+ href="#log4j-kafka-appender">Log4j Kafka Appender</a></li>
+        
+    
+
+            
+            </ul>
+        
+    
+
+            
+            </ul>
+        
+    </li>
+<li>
+          
+            <li><script>
+    function select(elem) {
+        var selectedClassName = "selected";
+        $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+        $(elem).addClass(selectedClassName);
+    }
+</script>
+
+<!-- 
+    this macro is to fetch the first child element that has url and return the url as the default one of the section
+    arguemnt:
+      ni: nav_item, corresponding to navigation item configured in mkdocs.yml's "pages"
+ -->
+
+
+<!-- 
+    this macro generates the style for indentation on sub tocs in side nav
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- 
+    this macro shows toc items including nested tocs, toc nesting level would depending on arguments described below
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+    <li class="toctree-l1 ">
+        <a class="" href="../applications/">Applications</a>
+        
+    </li>
+<li>
+          
+            <li><script>
+    function select(elem) {
+        var selectedClassName = "selected";
+        $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+        $(elem).addClass(selectedClassName);
+    }
+</script>
+
+<!-- 
+    this macro is to fetch the first child element that has url and return the url as the default one of the section
+    arguemnt:
+      ni: nav_item, corresponding to navigation item configured in mkdocs.yml's "pages"
+ -->
+
+
+<!-- 
+    this macro generates the style for indentation on sub tocs in side nav
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- 
+    this macro shows toc items including nested tocs, toc nesting level would depending on arguments described below
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+    <li class="toctree-l1 ">
+        <a class="" href="../developing-application/">Developing Application</a>
+        
+    </li>
+<li>
+          
+            <li><script>
+    function select(elem) {
+        var selectedClassName = "selected";
+        $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+        $(elem).addClass(selectedClassName);
+    }
+</script>
+
+<!-- 
+    this macro is to fetch the first child element that has url and return the url as the default one of the section
+    arguemnt:
+      ni: nav_item, corresponding to navigation item configured in mkdocs.yml's "pages"
+ -->
+
+
+<!-- 
+    this macro generates the style for indentation on sub tocs in side nav
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- 
+    this macro shows toc items including nested tocs, toc nesting level would depending on arguments described below
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+    <li class="toctree-l1 ">
+        <a class="" href="../underlying-design/">Underlying Design</a>
+        
+    </li>
+<li>
+          
+            <li><script>
+    function select(elem) {
+        var selectedClassName = "selected";
+        $(".wy-menu-vertical li.selected").removeClass(selectedClassName);
+        $(elem).addClass(selectedClassName);
+    }
+</script>
+
+<!-- 
+    this macro is to fetch the first child element that has url and return the url as the default one of the section
+    arguemnt:
+      ni: nav_item, corresponding to navigation item configured in mkdocs.yml's "pages"
+ -->
+
+
+<!-- 
+    this macro generates the style for indentation on sub tocs in side nav
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- 
+    this macro shows toc items including nested tocs, toc nesting level would depending on arguments described below
+    argument:
+      current_level: the level number of current toc item
+      max_level: a number indicating how many toc item levels should show up
+ -->
+
+
+<!-- generate side nav based on navigation configured in mkdocs.yml -->
+
+    <li class="toctree-l1 ">
+        <a class="" href="../reference/">Reference</a>
+        
+    </li>
+<li>
+          
+        </ul>
+      </div>
+      &nbsp;
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="..">Apache Eagle Documentation</a>
+      </nav>
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="..">Docs</a> &raquo;</li>
+    
+      
+    
+    <li>Using Eagle</li>
+    <li class="wy-breadcrumbs-aside">
+      
+        
+          <a href="https://github.com/apache/eagle/tree/master/docs" class="icon icon-github"> Edit on GitHub</a>
+        
+      
+    </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main">
+            <div class="section">
+              
+                <h1 id="manage-eagle-and-services">Manage Eagle and Services</h1>
+<ul>
+<li>
+<p>After Apache Eagle has been deployed (please reference <a href="../getting-started/#deployment"><strong>deployment</strong></a>), you can enter deployment directory and use commands below to control Apache Eagle Server.</p>
+<pre><code>./bin/eagle-server.sh start|stop|status
+</code></pre>
+</li>
+<li>
+<p>After starting the Eagle server, please type http://<EAGLE_SERVER_HOST>:<PORT>/ to open the web ui of Eagle.</p>
+</li>
+</ul>
+<hr />
+<h1 id="use-eagle-web-interface">Use Eagle Web Interface</h1>
+<ul>
+<li>
+<p>This is the typical Web Interface (short for WI) after setting up your Eagle monitoring environment. WI majorly contain the right main panel and left function menu.</p>
+<p><img alt="Eagle Web Interface" src="../include/images/eagle_web_interface.png" /></p>
+</li>
+</ul>
+<h2 id="home">Home</h2>
+<ul>
+<li>
+<p>This is the aggregated UI for configured sites, and the applications. It will show those created sites created, how many application installed for each sites, and alerts generated from that cluster. You can click “More info” link to view the details for particular site.</p>
+</li>
+<li>
+<p>The “<strong>Widgets</strong>” section is customizable; if the application developer have its application registered to Home page, you can find that in “<strong>Widgets</strong>” section. Please check the application developer guide about how to register applications to home widgets. It give you a shortcut to go directly to the application home.</p>
+</li>
+</ul>
+<h2 id="alert">Alert</h2>
+<ul>
+<li>In Alert menu, you can define the policies, list the policies and check your alerts there. </li>
+</ul>
+<h2 id="integration">Integration</h2>
+<ul>
+<li>The integration page provides the management functionality for Eagle. You can list the built-in applications there, create sites, and manage the applications in your site.</li>
+</ul>
+<h2 id="sites">Sites</h2>
+<ul>
+<li>It also gives you a shortcut to particular site.</li>
+</ul>
+<hr />
+<h1 id="setup-the-monitoring-application">Setup The Monitoring Application</h1>
+<h2 id="monitoring-applications">Monitoring Applications</h2>
+<ul>
+<li>
+<p>Eagle has an extensible framework to dynamically add new monitoring applications in Eagle environment. It also ships some built-in big data monitoring applications.</p>
+</li>
+<li>
+<p>Go to “<strong>Integration</strong>” -&gt; “<strong>Applications</strong>”, it will list a set of available monitoring applications which you can choose to monitor your services.</p>
+<p><img alt="Integation Applications" src="../include/images/integration_applications.png" /></p>
+</li>
+<li>
+<p>The “<strong>Application</strong>” column is the display name for an application, “<strong>Streams</strong>” is a logical name for the data stream from the monitored source after pre-processing, which will consumed by Alert Engine.</p>
+</li>
+<li>
+<p>At the moment, we have the below built-in applications shipped with Apache Eagle. You can refer to the application documentation to understand how to do the configuration for each monitoring application.</p>
+<table>
+<thead>
+<tr>
+<th>Application</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>Topology Health Check</td>
+<td>This application can be used to monitor the service healthiness for HDFS, HBase and YARN. You can get alerted once the master role or the slave role got crashed.</td>
+</tr>
+<tr>
+<td>Hadoop JMX Metrics Monitoring</td>
+<td>This application can be used to monitor the JMX metrics data from the master nodes of HDFS, HBase and YARN, e.g. NameNode, HBase Master and YARN Resource Manager.</td>
+</tr>
+<tr>
+<td>HDFS Audit Log Monitor</td>
+<td>This application can be used to monitor the data operations in HDFS, to detect sensitive data access and malicious operations; to protect from data leak or data loss.</td>
+</tr>
+<tr>
+<td>HBase Audit Log Monitor</td>
+<td>Same as HDFS Audit Log Monitor, this application is used to monitor the data operations in HBase.</td>
+</tr>
+<tr>
+<td>Map Reduce History Job</td>
+<td>This application is used to get the MapReduce history job counters from YARN history server and job running history from HDFS log directory.</td>
+</tr>
+<tr>
+<td>Map Reduce Running Job</td>
+<td>This application is used to get the MapReduce running job counter information using YARN Rest API.</td>
+</tr>
+<tr>
+<td>Hadoop Queue Monitor</td>
+<td>This application is used to get the resource scheduling and utilization info from YARN.</td>
+</tr>
+<tr>
+<td>MR Metrics Aggregation</td>
+<td>This application is used to aggregate the job counters and some resource utilization in a certain period of time (daily, weekly or monthly).</td>
+</tr>
+<tr>
+<td>Job Performance Monitor Web</td>
+<td>This application only contains the frontend, and depends on Map Reduce History Job and Map Reduce Running Job.</td>
+</tr>
+<tr>
+<td>Alert Engine</td>
+<td>Alert Engine is a special application and used to process the output data from other applications.</td>
+</tr>
+</tbody>
+</table>
+</li>
+</ul>
+<h2 id="managing-sites">Managing Sites</h2>
+<p>To enable a real monitoring use case, you have to create a site first, and install a certain application for this site, and finally start the application. We use site concept to group the running applications and avoid the application conflict.</p>
+<h3 id="sites_1">Sites</h3>
+<ul>
+<li>
+<p>Go to “<strong>Integration</strong>” -&gt; “<strong>Sites</strong>”, there will be a table listing the managed sites.</p>
+<p><img alt="Integration Sites" src="../include/images/integration_sites.png" /></p>
+</li>
+</ul>
+<h3 id="create-site">Create Site</h3>
+<ul>
+<li>
+<p>Click “<strong>New Site</strong>” on the bottom right of the Sites page. You can fill the information in site creation dialog.</p>
+<p><img alt="New Site" src="../include/images/new_site.png" /></p>
+</li>
+<li>
+<p>The “<strong>Site Id</strong>” should not be duplicated. After the creation, you can find it in sites page.</p>
+<p><img alt="Site List" src="../include/images/site_list.png" /></p>
+</li>
+</ul>
+<h3 id="configuring-a-site">Configuring a Site</h3>
+<ul>
+<li>
+<p>By clicking “<strong>Edit</strong>” button or the Site column in Sites table, you can have the Site configuration page, there you can install monitoring applications.</p>
+<p><img alt="Configure Site" src="../include/images/configure_site.png" /></p>
+</li>
+</ul>
+<h2 id="install-and-run-applications-in-site">Install and Run Applications in Site</h2>
+<ul>
+<li>
+<p>Choose the particular application which you want to install, you probably have something to fill, e.g. the HDFS NameNode address, Zookeeper address and port. Please check each application documentation for how to configure each application. </p>
+</li>
+<li>
+<p>After doing the installation, you can start the application by clicking <img alt="Start Icon" src="../include/images/start_icon.png" /> or stop the application by <img alt="Stop Icon" src="../include/images/stop_icon.png" />. You can check the “<strong>Status</strong>” column about the running status. Usually, it should have “<strong>INITIALIZED</strong>” or “<strong>RUNNING</strong>” for a healthy application.</p>
+</li>
+</ul>
+<hr />
+<h1 id="define-policies">Define Policies</h1>
+<p>After setting up the monitoring applications, you probably want to setup some alert policies against the monitored data, so you can get notified once any violation on the data. Eagle has a centralized place for policy definition.</p>
+<h2 id="policies">Policies</h2>
+<ul>
+<li>
+<p>Go to “<strong>Alert</strong>” -&gt; “<strong>Policies</strong>”, you can check the policies defined and take control on whether to enable the policy:</p>
+<p><img alt="Alert Policies" src="../include/images/alert_policies.png" /></p>
+</li>
+<li>
+<p>You can apply the below actions for a certain policy:</p>
+<ul>
+<li>
+<p><img alt="Start Icon" src="../include/images/start_icon.png" />: enable a policy</p>
+</li>
+<li>
+<p><img alt="Stop Icon" src="../include/images/stop_icon.png" />: disable a policy</p>
+</li>
+<li>
+<p><img alt="Edit Icon" src="../include/images/edit_icon.png" />: edit a policy</p>
+</li>
+<li>
+<p><img alt="Delete Icon" src="../include/images/delete_icon.png" />: purge a policy</p>
+</li>
+</ul>
+</li>
+</ul>
+<h2 id="define-or-edit-policies">Define or Edit Policies</h2>
+<ul>
+<li>
+<p>If you want to create a new policy, click “<strong>Alert</strong>” -&gt; “<strong>Define Policy</strong>”, or you can enter into the policy definition page by editing an existing policy. After that, you can go to the policy list to enable the policy dynamically.</p>
+<p><img alt="Define Policy" src="../include/images/alert_define_policy.png" /></p>
+</li>
+</ul>
+<h3 id="source-stream">Source Stream</h3>
+<ul>
+<li>The source stream gives user a full view about what data stream is available for application defined for particular site, as well as the data structures in each data stream. Data stream name is suffixed by the site name.</li>
+</ul>
+<h3 id="policy-name">Policy Name</h3>
+<ul>
+<li>The policy name should be globally unique.</li>
+</ul>
+<h3 id="publish-alerts">Publish Alerts</h3>
+<ul>
+<li>
+<p>In this section, you can define the alert publishment method by clicking the “<strong>+Add Publisher</strong>”.</p>
+<p><img alt="Add Publisher" src="../include/images/add_publisher.png" /></p>
+</li>
+<li>
+<p>You can choose the publishment method from an existing policy or by creating new publisher. </p>
+</li>
+<li>
+<p>There are four built-in publisher types:</p>
+<ul>
+<li>
+<p><strong>EmailPublisher</strong>: org.apache.eagle.alert.engine.publisher.impl.AlertEmailPublisher</p>
+</li>
+<li>
+<p><strong>KafkaPublisher</strong>: org.apache.eagle.alert.engine.publisher.impl.AlertKafkaPublisher</p>
+</li>
+<li>
+<p><strong>SlackPublisher</strong>: org.apache.eagle.alert.engine.publisher.impl.AlertSlackPublisher</p>
+</li>
+<li>
+<p><strong>EagleStoragePlugin</strong>: org.apache.eagle.alert.engine.publisher.impl.AlertEagleStoragePlugin</p>
+</li>
+</ul>
+</li>
+</ul>
+<h3 id="policy-syntax">Policy Syntax</h3>
+<ul>
+<li>
+<p>Currently, we support SiddhiQL(please view Siddhi Query Language Specification <a href="https://docs.wso2.com/display/CEP300/Siddhi+Language+Specification">here</a>)</p>
+</li>
+<li>
+<p>In order to explain how stream data is processed, let us take policy below as an example:</p>
+<pre><code>from map_reduce_failed_job_stream[site=="sandbox" and currentState=="FAILED"]
+select * group by jobId insert into map_reduce_failed_job_stream_out
+</code></pre>
+</li>
+<li>
+<p>This policy contains below parts:</p>
+<ul>
+<li>
+<p><strong>Source</strong>: from map_reduce_failed_job_stream</p>
+</li>
+<li>
+<p><strong>Filter</strong>: [site=="sandbox" and currentState=="FAILED"]</p>
+</li>
+<li>
+<p><strong>Projection</strong>: select *</p>
+</li>
+<li>
+<p><strong>GroupBy</strong>: group by jobId</p>
+</li>
+<li>
+<p><strong>Destination</strong>: insert into map_reduce_failed_job_stream_out</p>
+</li>
+</ul>
+</li>
+<li>
+<p>Source Streams(schema) are defined by applications, and applications will write stream data to data sink(currently, we support kafka as data sink).</p>
+<pre><code>&lt;streams&gt;
+    &lt;stream&gt;
+        &lt;streamId&gt;map_reduce_failed_job_stream&lt;/streamId&gt;
+        &lt;description&gt;Map Reduce Failed Job Stream&lt;/description&gt;
+        &lt;validate&gt;true&lt;/validate&gt;
+        &lt;columns&gt;
+            &lt;column&gt;
+                &lt;name&gt;site&lt;/name&gt;
+                &lt;type&gt;string&lt;/type&gt;
+            &lt;/column&gt;
+            …...
+            &lt;column&gt;
+                &lt;name&gt;jobId&lt;/name&gt;
+                &lt;type&gt;string&lt;/type&gt;
+            &lt;column&gt;
+                &lt;name&gt;currentState&lt;/name&gt;
+                &lt;type&gt;string&lt;/type&gt;
+            &lt;/column&gt;
+        &lt;/columns&gt;
+    &lt;/stream&gt;
+&lt;/streams&gt;
+</code></pre>
+</li>
+<li>
+<p>After policy is defined, Alert engine will create siddhi execution runtime for the policy(also load stream data schema from metadata store). Since siddhi execution runtime knows the stream data schema, then it will process stream data and do the calculation.</p>
+</li>
+</ul>
+<hr />
+<h1 id="monitoring-dashboard">Monitoring Dashboard</h1>
+<ul>
+<li>
+<p>After setting the sites and applications, you can find the site item from the home page or “Sites” menu.</p>
+</li>
+<li>
+<p>Here is a site home example. After entering the site home, the left menu will be replaced by application dashboard links only related to that site, so you can switch between the application dashboard quickly. In the right panel, it contains the application icons installed in this site, but depends on if the application has its dashboard defined. You can click the application icon or the application links to go to the application dashboard home. Please check the application documentation about how to use the application monitoring dashboard.</p>
+<p><img alt="Dashboard" src="../include/images/dashboard.png" /></p>
+</li>
+</ul>
+<hr />
+<h1 id="check-the-alerts">Check The Alerts</h1>
+<ul>
+<li>
+<p>Eagle has all the alerts generated by all the applications stored in its database, so you can check your application alerts from Eagle WI. </p>
+</li>
+<li>
+<p>Go to “<strong>Alert</strong>” -&gt; “<strong>Alerts</strong>”, you can find the alerts table.</p>
+<p><img alt="Alerts" src="../include/images/alert_alerts.png" /></p>
+</li>
+<li>
+<p>Also you can check more detailed information by clicking “<strong>Detail</strong>” link for each alert item.</p>
+<p><img alt="Alert Details" src="../include/images/alert_details.png" /></p>
+</li>
+</ul>
+<hr />
+<h1 id="how-to-stream-audit-log-into-kafka">How to stream audit log into Kafka</h1>
+<h2 id="logstash">Logstash</h2>
+<p>The sample configuration is tested with logstash-2.3.4. Logstash is required to be installed on the namenode host.</p>
+<ul>
+<li>
+<p><strong>Step 1</strong>: Create a Kafka topic as the streaming input.</p>
+<p>Here is an sample Kafka command to create topic 'sandbox_hdfs_audit_log'</p>
+<pre><code>cd &lt;kafka-home&gt;
+bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic sandbox_hdfs_audit_log
+</code></pre>
+</li>
+<li>
+<p><strong>Step 2</strong>: Create a Logstash configuration file under ${LOGSTASH_HOME}/conf. Here is a sample.</p>
+<pre><code>input {
+      file {
+          type =&gt; "hdp-nn-audit"
+          path =&gt; "/tmp/test/hdfs-audit.log"
+          start_position =&gt; end
+          sincedb_path =&gt; "/dev/null"
+       }
+  }
+ output {
+      if [type] == "hdp-nn-audit" {
+          kafka {
+            codec =&gt; plain {
+                format =&gt; "%{message}"
+            }
+            bootstrap_servers =&gt; "host:9092"
+            topic_id =&gt; "hdfs_audit_log"
+            acks =&gt; "0"
+            timeout_ms =&gt; 10000
+
+            send_buffer_bytes =&gt; 102400
+            client_id =&gt; "hdp-nn-audit"
+
+            workers =&gt; 10
+            compression_type =&gt; "gzip"
+         }
+          # stdout { codec =&gt; rubydebug }
+  }
+}
+</code></pre>
+</li>
+<li>
+<p><strong>Step 4</strong>: Start Logstash</p>
+<pre><code>bin/logstash -f conf/sample.conf
+</code></pre>
+</li>
+<li>
+<p><strong>Step 5</strong>: Check whether logs are flowing into the kafka topic specified by <code>topic_id</code></p>
+</li>
+</ul>
+<h2 id="filebeat">Filebeat</h2>
+<p>The sample filebeat.yml is tested with filebeat-5.0.0-beta1-linux-x86_64. The throughput can be up to 20K messages per second. Filebeat is required to be installed on the namenode host.</p>
+<pre><code>    filebeat.publish_async: false
+    filebeat.spool_size: 8192
+    filebeat.idle_timeout: 5s
+    max_procs: 1
+    queue_size: 1000
+
+    filebeat.prospectors:
+    - input_type: log
+      paths:
+         - /tmp/test/hdfs-audit.log
+      #tail_files: true
+      harvester_buffer_size: 8192
+
+    output.kafka:
+      enabled: true
+      hosts: ["host:9092"]
+      topic: "phx_hdfs_audit_log"
+      client_id: "client-host"
+      worker: 10
+      max_retries: 3
+      bulk_max_size: 8192
+      channel_buffer_size: 512
+      timeout: 10
+      broker_timeout: 3s
+      keep_alive: 0
+      compression: none
+      max_message_bytes: 1000000
+      required_acks: 0
+      flush_interval: 1
+
+    logging.metrics.period: 10s
+
+    processors:
+      - include_fields:
+         fields: ["message", "beat.hostname"]
+</code></pre>
+<h2 id="log4j-kafka-appender">Log4j Kafka Appender</h2>
+<p>This sample configuration is tested in HDP sandbox. <code>Restarting namenode is required</code> after updating the log4j configuration. </p>
+<ul>
+<li>
+<p><strong>Step 1</strong>: Create a Kafka topic. Here is an example Kafka command for creating topic "sandbox_hdfs_audit_log"</p>
+<pre><code>cd &lt;kafka-home&gt;
+bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic sandbox_hdfs_audit_log
+</code></pre>
+</li>
+<li>
+<p><strong>Step 2</strong>: Configure $HADOOP_CONF_DIR/log4j.properties, and add a log4j appender "KAFKA_HDFS_AUDIT" to hdfs audit logging</p>
+<pre><code>log4j.appender.KAFKA_HDFS_AUDIT=org.apache.eagle.log4j.kafka.KafkaLog4jAppender
+log4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log
+log4j.appender.KAFKA_HDFS_AUDIT.BrokerList=sandbox.hortonworks.com:6667
+log4j.appender.KAFKA_HDFS_AUDIT.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer
+log4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout
+log4j.appender.KAFKA_HDFS_AUDIT.Layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+log4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async
+#log4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1
+#log4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1
+</code></pre>
+</li>
+<li>
+<p><strong>Step 3</strong>: Edit $HADOOP_CONF_DIR/hadoop-env.sh, and add the reference to KAFKA_HDFS_AUDIT to HADOOP_NAMENODE_OPTS.</p>
+<pre><code>-Dhdfs.audit.logger=INFO,DRFAAUDIT,KAFKA_HDFS_AUDIT
+</code></pre>
+</li>
+<li>
+<p><strong>Step 4</strong>: Edit $HADOOP_CONF_DIR/hadoop-env.sh, and append the following command to it.</p>
+<pre><code>export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/path/to/eagle/lib/log4jkafka/lib/*
+</code></pre>
+</li>
+<li>
+<p><strong>Step 5</strong>: save the changes and restart the namenode.</p>
+</li>
+<li>
+<p><strong>Step 6</strong>: Check whether logs are flowing into Topic sandbox_hdfs_audit_log</p>
+<pre><code>$ /usr/hdp/current/kafka-broker/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic sandbox_hdfs_audit_log
+</code></pre>
+</li>
+</ul>
+              
+            </div>
+          </div>
+          <footer>
+  
+    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
+      
+        <a href="../applications/" class="btn btn-neutral float-right" title="Applications">Next <span class="icon icon-circle-arrow-right"></span></a>
+      
+      
+        <a href="../getting-started/" class="btn btn-neutral" title="Getting Started"><span class="icon icon-circle-arrow-left"></span> Previous</a>
+      
+    </div>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <!-- Copyright etc -->
+    
+  </div>
+
+  Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>
+	  
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+
+<div class="rst-versions" role="note" style="cursor: pointer">
+    <span class="rst-current-version" data-toggle="rst-current-version">
+      
+          <a href="https://github.com/apache/eagle/tree/master/docs" class="icon icon-github" style="float: left; color: #fcfcfc"> GitHub</a>
+      
+      
+        <span><a href="../getting-started/" style="color: #fcfcfc;">&laquo; Previous</a></span>
+      
+      
+        <span style="margin-left: 15px"><a href="../applications/" style="color: #fcfcfc">Next &raquo;</a></span>
+      
+    </span>
+</div>
+
+</body>
+</html>

Propchange: eagle/site/docs/latest/using-eagle/index.html
------------------------------------------------------------------------------
    svn:eol-style = native

Added: eagle/site/docs/v0.5.0/css/highlight.css
URL: http://svn.apache.org/viewvc/eagle/site/docs/v0.5.0/css/highlight.css?rev=1789966&view=auto
==============================================================================
--- eagle/site/docs/v0.5.0/css/highlight.css (added)
+++ eagle/site/docs/v0.5.0/css/highlight.css Mon Apr  3 11:35:50 2017
@@ -0,0 +1,125 @@
+/*
+This is the GitHub theme for highlight.js
+
+github.com style (c) Vasily Polovnyov <vast@whiteants.net>
+
+*/
+
+.hljs {
+  display: block;
+  overflow-x: auto;
+  padding: 0.5em;
+  color: #333;
+  -webkit-text-size-adjust: none;
+}
+
+.hljs-comment,
+.diff .hljs-header,
+.hljs-javadoc {
+  color: #998;
+  font-style: italic;
+}
+
+.hljs-keyword,
+.css .rule .hljs-keyword,
+.hljs-winutils,
+.nginx .hljs-title,
+.hljs-subst,
+.hljs-request,
+.hljs-status {
+  color: #333;
+  font-weight: bold;
+}
+
+.hljs-number,
+.hljs-hexcolor,
+.ruby .hljs-constant {
+  color: #008080;
+}
+
+.hljs-string,
+.hljs-tag .hljs-value,
+.hljs-phpdoc,
+.hljs-dartdoc,
+.tex .hljs-formula {
+  color: #d14;
+}
+
+.hljs-title,
+.hljs-id,
+.scss .hljs-preprocessor {
+  color: #900;
+  font-weight: bold;
+}
+
+.hljs-list .hljs-keyword,
+.hljs-subst {
+  font-weight: normal;
+}
+
+.hljs-class .hljs-title,
+.hljs-type,
+.vhdl .hljs-literal,
+.tex .hljs-command {
+  color: #458;
+  font-weight: bold;
+}
+
+.hljs-tag,
+.hljs-tag .hljs-title,
+.hljs-rule .hljs-property,
+.django .hljs-tag .hljs-keyword {
+  color: #000080;
+  font-weight: normal;
+}
+
+.hljs-attribute,
+.hljs-variable,
+.lisp .hljs-body,
+.hljs-name {
+  color: #008080;
+}
+
+.hljs-regexp {
+  color: #009926;
+}
+
+.hljs-symbol,
+.ruby .hljs-symbol .hljs-string,
+.lisp .hljs-keyword,
+.clojure .hljs-keyword,
+.scheme .hljs-keyword,
+.tex .hljs-special,
+.hljs-prompt {
+  color: #990073;
+}
+
+.hljs-built_in {
+  color: #0086b3;
+}
+
+.hljs-preprocessor,
+.hljs-pragma,
+.hljs-pi,
+.hljs-doctype,
+.hljs-shebang,
+.hljs-cdata {
+  color: #999;
+  font-weight: bold;
+}
+
+.hljs-deletion {
+  background: #fdd;
+}
+
+.hljs-addition {
+  background: #dfd;
+}
+
+.diff .hljs-change {
+  background: #0086b3;
+}
+
+.hljs-chunk {
+  color: #aaa;
+}

Propchange: eagle/site/docs/v0.5.0/css/highlight.css
------------------------------------------------------------------------------
    svn:eol-style = native

Added: eagle/site/docs/v0.5.0/include/images/hdfs_install_2.png
URL: http://svn.apache.org/viewvc/eagle/site/docs/v0.5.0/include/images/hdfs_install_2.png?rev=1789966&view=auto
==============================================================================
Binary file - no diff available.

Propchange: eagle/site/docs/v0.5.0/include/images/hdfs_install_2.png
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: eagle/site/docs/v0.5.0/include/images/hdfs_install_3.png
URL: http://svn.apache.org/viewvc/eagle/site/docs/v0.5.0/include/images/hdfs_install_3.png?rev=1789966&view=auto
==============================================================================
Binary file - no diff available.

Propchange: eagle/site/docs/v0.5.0/include/images/hdfs_install_3.png
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: eagle/site/docs/v0.5.0/include/images/hdfs_policy_1.png
URL: http://svn.apache.org/viewvc/eagle/site/docs/v0.5.0/include/images/hdfs_policy_1.png?rev=1789966&view=auto
==============================================================================
Binary file - no diff available.

Propchange: eagle/site/docs/v0.5.0/include/images/hdfs_policy_1.png
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: eagle/site/docs/v0.5.0/include/images/health_check_installation.png
URL: http://svn.apache.org/viewvc/eagle/site/docs/v0.5.0/include/images/health_check_installation.png?rev=1789966&view=auto
==============================================================================
Binary file - no diff available.

Propchange: eagle/site/docs/v0.5.0/include/images/health_check_installation.png
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: eagle/site/docs/v0.5.0/include/images/health_check_policy.png
URL: http://svn.apache.org/viewvc/eagle/site/docs/v0.5.0/include/images/health_check_policy.png?rev=1789966&view=auto
==============================================================================
Binary file - no diff available.

Propchange: eagle/site/docs/v0.5.0/include/images/health_check_policy.png
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: eagle/site/docs/v0.5.0/include/images/health_check_settings.png
URL: http://svn.apache.org/viewvc/eagle/site/docs/v0.5.0/include/images/health_check_settings.png?rev=1789966&view=auto
==============================================================================
Binary file - no diff available.

Propchange: eagle/site/docs/v0.5.0/include/images/health_check_settings.png
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: eagle/site/docs/v0.5.0/include/images/health_check_stream.png
URL: http://svn.apache.org/viewvc/eagle/site/docs/v0.5.0/include/images/health_check_stream.png?rev=1789966&view=auto
==============================================================================
Binary file - no diff available.

Propchange: eagle/site/docs/v0.5.0/include/images/health_check_stream.png
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: eagle/site/docs/v0.5.0/js/highlight.pack.js
URL: http://svn.apache.org/viewvc/eagle/site/docs/v0.5.0/js/highlight.pack.js?rev=1789966&view=auto
==============================================================================
--- eagle/site/docs/v0.5.0/js/highlight.pack.js (added)
+++ eagle/site/docs/v0.5.0/js/highlight.pack.js Mon Apr  3 11:35:50 2017
@@ -0,0 +1,2 @@

[... 4 lines stripped ...]
Propchange: eagle/site/docs/v0.5.0/js/highlight.pack.js
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message