apex-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t..@apache.org
Subject [12/16] apex-site git commit: from fe593b313add7ab1f8bb0b6b51603dbef88003b1
Date Tue, 28 Mar 2017 06:34:14 GMT
http://git-wip-us.apache.org/repos/asf/apex-site/blob/afbb4705/content/docs/malhar-3.7/fonts/fontawesome-webfont.woff
----------------------------------------------------------------------
diff --git a/content/docs/malhar-3.7/fonts/fontawesome-webfont.woff b/content/docs/malhar-3.7/fonts/fontawesome-webfont.woff
new file mode 100755
index 0000000..b9bd17e
Binary files /dev/null and b/content/docs/malhar-3.7/fonts/fontawesome-webfont.woff differ

http://git-wip-us.apache.org/repos/asf/apex-site/blob/afbb4705/content/docs/malhar-3.7/footer.html
----------------------------------------------------------------------
diff --git a/content/docs/malhar-3.7/footer.html b/content/docs/malhar-3.7/footer.html
new file mode 100644
index 0000000..1103193
--- /dev/null
+++ b/content/docs/malhar-3.7/footer.html
@@ -0,0 +1,23 @@
+<footer>
+  {% if next_page or previous_page %}
+    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
+      {% if next_page %}
+        <a href="{{ next_page.url }}" class="btn btn-neutral float-right" title="{{ next_page.title
}}">Next <span class="icon icon-circle-arrow-right"></span></a>
+      {% endif %}
+      {% if previous_page %}
+        <a href="{{ previous_page.url }}" class="btn btn-neutral" title="{{ previous_page.title
}}"><span class="icon icon-circle-arrow-left"></span> Previous</a>
+      {% endif %}
+    </div>
+  {% endif %}
+
+  <hr/>
+
+  <div role="contentinfo">
+    <!-- Copyright etc -->
+    {% if copyright %}
+      <p>{{ copyright }}</p>
+    {% endif %}
+  </div>
+
+  Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a>
provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>

http://git-wip-us.apache.org/repos/asf/apex-site/blob/afbb4705/content/docs/malhar-3.7/images/malhar-operators.png
----------------------------------------------------------------------
diff --git a/content/docs/malhar-3.7/images/malhar-operators.png b/content/docs/malhar-3.7/images/malhar-operators.png
new file mode 100644
index 0000000..ac09622
Binary files /dev/null and b/content/docs/malhar-3.7/images/malhar-operators.png differ

http://git-wip-us.apache.org/repos/asf/apex-site/blob/afbb4705/content/docs/malhar-3.7/img/favicon.ico
----------------------------------------------------------------------
diff --git a/content/docs/malhar-3.7/img/favicon.ico b/content/docs/malhar-3.7/img/favicon.ico
new file mode 100644
index 0000000..e85006a
Binary files /dev/null and b/content/docs/malhar-3.7/img/favicon.ico differ

http://git-wip-us.apache.org/repos/asf/apex-site/blob/afbb4705/content/docs/malhar-3.7/index.html
----------------------------------------------------------------------
diff --git a/content/docs/malhar-3.7/index.html b/content/docs/malhar-3.7/index.html
new file mode 100644
index 0000000..232ef52
--- /dev/null
+++ b/content/docs/malhar-3.7/index.html
@@ -0,0 +1,396 @@
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  
+  
+  <title>Apache Apex Malhar Documentation</title>
+  
+
+  <link rel="shortcut icon" href="favicon.ico">
+  
+
+  
+  <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700'
rel='stylesheet' type='text/css'>
+
+  <link rel="stylesheet" href="./css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="./css/theme_extra.css" type="text/css" />
+  <link rel="stylesheet" href="./css/highlight.css">
+
+  
+  <script>
+    // Current page data
+    var mkdocs_page_name = null;
+    var mkdocs_page_input_path = "index.md";
+    var mkdocs_page_url = "/";
+  </script>
+  
+  <script src="./js/jquery-2.1.1.min.js"></script>
+  <script src="./js/modernizr-2.8.3.min.js"></script>
+  <script type="text/javascript" src="./js/highlight.pack.js"></script>
+  <script src="./js/theme.js"></script> 
+
+  
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+      <div class="wy-side-nav-search">
+        <a href="." class="icon icon-home"> Apache Apex Malhar Documentation</a>
+        <div role="search">
+  <form id ="rtd-search-form" class="wy-form" action="./search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+  </form>
+</div>
+      </div>
+
+      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main
navigation">
+        <ul class="current">
+          
+            <li>
+    <li class="toctree-l1 current">
+        <a class="current" href=".">Apache Apex Malhar</a>
+        
+            <ul>
+            
+                <li class="toctree-l3"><a href="#apache-apex-malhar">Apache Apex
Malhar</a></li>
+                
+            
+                <li class="toctree-l3"><a href="#capabilities-common-across-malhar-operators">Capabilities
common across Malhar operators</a></li>
+                
+            
+                <li class="toctree-l3"><a href="#operator-library-overview">Operator
Library Overview</a></li>
+                
+                    <li><a class="toctree-l4" href="#inputoutput-connectors">Input/output
connectors</a></li>
+                
+                    <li><a class="toctree-l4" href="#parsers">Parsers</a></li>
+                
+                    <li><a class="toctree-l4" href="#stream-manipulation">Stream
manipulation</a></li>
+                
+                    <li><a class="toctree-l4" href="#compute">Compute</a></li>
+                
+                    <li><a class="toctree-l4" href="#languages-support">Languages
Support</a></li>
+                
+            
+            </ul>
+        
+    </li>
+<li>
+          
+            <li>
+    <ul class="subnav">
+    <li><span>APIs</span></li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="apis/calcite/">SQL</a>
+        
+    </li>
+
+        
+    </ul>
+<li>
+          
+            <li>
+    <ul class="subnav">
+    <li><span>Operators</span></li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/block_reader/">Block Reader</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/csvformatter/">CSV Formatter</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/csvParserOperator/">CSV Parser</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/deduper/">Deduper</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/enricher/">Enricher</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/fsInputOperator/">File Input</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/file_output/">File Output</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/file_splitter/">File Splitter</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/filter/">Filter</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/AbstractJdbcTransactionableOutputOperator/">Jdbc
Output Operator</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/jdbcPollInputOperator/">JDBC Poller Input</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/jmsInputOperator/">JMS Input</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/jsonFormatter/">JSON Formatter</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/jsonParser/">JSON Parser</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/kafkaInputOperator/">Kafka Input</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/transform/">Transformer</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/windowedOperator/">Windowed Operator</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/xmlParserOperator/">XML Parser</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/jsonParser/">Json Parser</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/jsonFormatter/">Json Formatter</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/transform/">Transform Operator</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/csvformatter/">Csv Formatter</a>
+        
+    </li>
+
+        
+            
+    <li class="toctree-l1 ">
+        <a class="" href="operators/s3outputmodule/">S3 Output Module</a>
+        
+    </li>
+
+        
+    </ul>
+<li>
+          
+        </ul>
+      </div>
+      &nbsp;
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href=".">Apache Apex Malhar Documentation</a>
+      </nav>
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href=".">Docs</a> &raquo;</li>
+    
+      
+    
+    <li>Apache Apex Malhar</li>
+    <li class="wy-breadcrumbs-aside">
+      
+    </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main">
+            <div class="section">
+              
+                <h1 id="apache-apex-malhar">Apache Apex Malhar</h1>
+<p>Apache Apex Malhar is an open source operator and codec library that can be used
with the <a href="http://apex.apache.org/">Apache Apex</a> platform to build real-time
streaming applications.  Enabling users to extract value quickly, Malhar operators help get
data in, analyze it in real-time, and get data out of Hadoop.  In addition to the operators,
the library contains a number of example applications, demonstrating operator features and
capabilities.</p>
+<p><img alt="MalharDiagram" src="./images/malhar-operators.png" /></p>
+<h1 id="capabilities-common-across-malhar-operators">Capabilities common across Malhar
operators</h1>
+<p>For most streaming platforms, connectors are afterthoughts and often end up being
simple ‘bolt-ons’ to the platform. As a result they often cause performance issues or
data loss when put through failure scenarios and scalability requirements. Malhar operators
do not face these issues as they were designed to be integral parts of Apex. Hence, they have
following core streaming runtime capabilities</p>
+<ol>
+<li><strong>Fault tolerance</strong> – Malhar operators where applicable
have fault tolerance built in. They use the checkpoint capability provided by the framework
to ensure that there is no data loss under ANY failure scenario.</li>
+<li><strong>Processing guarantees</strong> – Malhar operators where applicable
provide out of the box support for ALL three processing guarantees – exactly once, at-least
once, and at-most once WITHOUT requiring the user to write any additional code.  Some operators,
like MQTT operator, deal with source systems that can not track processed data and hence need
the operators to keep track of the data.  Malhar has support for a generic operator that uses
alternate storage like HDFS to facilitate this.  Finally for databases that support transactions
or support any sort of atomic batch operations Malhar operators can do exactly once down to
the tuple level.</li>
+<li><strong>Dynamic updates</strong> – Based on changing business conditions
you often have to tweak several parameters used by the operators in your streaming application
without incurring any application downtime. You can also change properties of a Malhar operator
at runtime without having to bring down the application.</li>
+<li><strong>Ease of extensibility</strong> – Malhar operators are based
on templates that are easy to extend.</li>
+<li><strong>Partitioning support</strong> – In streaming applications
the input data stream often needs to be partitioned based on the contents of the stream. Also
for operators that ingest data from external systems partitioning needs to be done based on
the capabilities of the external system.  For example with Kafka, the operator can automatically
scale up or down based on the changes in the number of Kafka partitions.</li>
+</ol>
+<h1 id="operator-library-overview">Operator Library Overview</h1>
+<h2 id="inputoutput-connectors">Input/output connectors</h2>
+<p>Below is a summary of the various sub categories of input and output operators.
Input operators also have a corresponding output operator</p>
+<ul>
+<li><strong>File Systems</strong> – Most streaming analytics use cases
require the data to be stored in HDFS or perhaps S3 if the application is running in AWS.
 Users often need to re-run their streaming analytical applications against historical data
or consume data from upstream processes that are perhaps writing to some NFS share.  Apex
supports input &amp; output operators for HDFS, S3, NFS &amp; Local Files.  There
are also File Splitter and Block Reader operators, which can accelecate processing of large
files by splitting and paralellizing the work across non-overlapping sets of file blocks.</li>
+<li><strong>Relational Databases</strong> – Most stream processing use
cases require some reference data lookups to enrich, tag or filter streaming data. There is
also a need to save results of the streaming analytical computation to a database so an operational
dashboard can see them. Apex supports a JDBC operator so you can read/write data from any
JDBC compliant RDBMS like Oracle, MySQL, Sqlite, etc.</li>
+<li><strong>NoSQL Databases</strong> – NoSQL key-value pair databases
like Cassandra &amp; HBase are a common part of streaming analytics application architectures
to lookup reference data or store results.  Malhar has operators for HBase, Cassandra, Accumulo,
Aerospike, MongoDB, and CouchDB.</li>
+<li><strong>Messaging Systems</strong> – Kafka, JMS, and similar systems
are the workhorses of messaging infrastructure in most enterprises.  Malhar has a robust,
industry-tested set of operators to read and write Kafka, JMS, ZeroMQ, and RabbitMQ messages.</li>
+<li><strong>Notification Systems</strong> – Malhar includes an operator
for sending notifications via SMTP.</li>
+<li><strong>In-memory Databases &amp; Caching platforms</strong> -
Some streaming use cases need instantaneous access to shared state across the application.
Caching platforms and in-memory databases serve this purpose really well. To support these
use cases, Malhar has operators for memcached and Redis.</li>
+<li><strong>Social Media</strong> - Malhar includes an operator to connect
to the popular Twitter stream fire hose.</li>
+<li><strong>Protocols</strong> - Malhar provides connectors that can communicate
in HTTP, RSS, Socket, WebSocket, FTP, and MQTT.</li>
+</ul>
+<h2 id="parsers">Parsers</h2>
+<p>There are many industry vertical specific data formats that a streaming application
developer might need to parse. Often there are existing parsers available for these that can
be directly plugged into an Apache Apex application. For example in the Telco space, a Java
based CDR parser can be directly plugged into Apache Apex operator. To further simplify development
experience, Malhar also provides some operators for parsing common formats like XML (DOM &amp;
SAX), JSON (flat map converter), Apache log files, syslog, etc.</p>
+<h2 id="stream-manipulation">Stream manipulation</h2>
+<p>Streaming data inevitably needs processing to clean, filter, tag, summarize, etc.
The goal of Malhar is to enable the application developer to focus on WHAT needs to be done
to the stream to get it in the right format and not worry about the HOW.  Malhar has several
operators to perform the common stream manipulation actions like – GroupBy, Join, Distinct/Unique,
Limit, OrderBy, Split, Sample, Inner join, Outer join, Select, Update etc.</p>
+<h2 id="compute">Compute</h2>
+<p>One of the most important promises of a streaming analytics platform like Apache
Apex is the ability to do analytics in real-time. However delivering on the promise becomes
really difficult when the platform does not provide out of the box operators to support variety
of common compute functions as the user then has to worry about making these scalable, fault
tolerant, stateful, etc.  Malhar takes this responsibility away from the application developer
by providing a variety of out of the box computational operators.</p>
+<p>Below is just a snapshot of the compute operators available in Malhar</p>
+<ul>
+<li>Statistics and math - Various mathematical and statistical computations over application
defined time windows.</li>
+<li>Filtering and pattern matching</li>
+<li>Sorting, maps, frequency, TopN, BottomN</li>
+<li>Random data generators</li>
+</ul>
+<h2 id="languages-support">Languages Support</h2>
+<p>Migrating to a new platform often requires re-use of the existing code that would
be difficult or time-consuming to re-write.  With this in mind, Malhar supports invocation
of code written in other languages by wrapping them in one of the library operators, and allows
execution of software written in:</p>
+<ul>
+<li>JavaScript</li>
+<li>Python</li>
+<li>R</li>
+<li>Ruby</li>
+</ul>
+              
+            </div>
+          </div>
+          <footer>
+  
+    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
+      
+        <a href="apis/calcite/" class="btn btn-neutral float-right" title="SQL">Next
<span class="icon icon-circle-arrow-right"></span></a>
+      
+      
+    </div>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <!-- Copyright etc -->
+    
+  </div>
+
+  Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a>
provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>
+	  
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+
+<div class="rst-versions" role="note" style="cursor: pointer">
+    <span class="rst-current-version" data-toggle="rst-current-version">
+      
+      
+      
+        <span style="margin-left: 15px"><a href="apis/calcite/" style="color: #fcfcfc">Next
&raquo;</a></span>
+      
+    </span>
+</div>
+
+</body>
+</html>
+
+<!--
+MkDocs version : 0.15.3
+Build Date UTC : 2017-03-28 06:30:43.879102
+-->


Mime
View raw message