flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From u..@apache.org
Subject [27/51] [partial] flink-web git commit: [hotfix] Manual build of docs
Date Tue, 30 Jun 2015 10:15:57 GMT
http://git-wip-us.apache.org/repos/asf/flink-web/blob/396616d4/content/docs/0.9/libs/ml/quickstart.html
----------------------------------------------------------------------
diff --git a/content/docs/0.9/libs/ml/quickstart.html b/content/docs/0.9/libs/ml/quickstart.html
new file mode 100644
index 0000000..188864b
--- /dev/null
+++ b/content/docs/0.9/libs/ml/quickstart.html
@@ -0,0 +1,438 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE html>
+
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+    
+    <title>Apache Flink 0.9.0 Documentation: FlinkML - Quickstart Guide</title>
+    
+    <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+    <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+
+    <!-- Bootstrap -->
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
+    <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css">
+    <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css">
+    <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css">
+    
+    <script type="text/x-mathjax-config">
+        MathJax.Hub.Config({
+        tex2jax: {
+          inlineMath: [['$','$'], ['\\(','\\)']] },
+        TeX: {
+          equationNumbers: { autoNumber: "AMS" } }
+        });
+    </script>
+    <script type="text/javascript"
+            src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
+    </script>
+    
+    <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+    <!--[if lt IE 9]>
+      <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
+      <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+    <![endif]-->
+  </head>
+  <body>
+    
+    
+
+
+
+
+    <!-- Top navbar. -->
+    <nav class="navbar navbar-default navbar-fixed-top">
+      <div class="container">
+        <!-- The logo. -->
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <div class="navbar-logo">
+            <a href="http://flink.apache.org"><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"></a>
+          </div>
+        </div><!-- /.navbar-header -->
+
+        <!-- The navigation links. -->
+        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+          <ul class="nav navbar-nav">
+            <li><a href="http://flink.apache.org/docs/0.9/index.html">Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li>
+
+            <!-- Setup -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/setup" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a>
+              <ul class="dropdown-menu" role="menu">
+                <li><a href="http://flink.apache.org/docs/0.9/setup/building.html">Get Flink 0.9-SNAPSHOT</a></li>
+
+                <li class="divider"></li>
+                <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html" class="active">Local</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html">Cluster (Standalone)</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html">YARN</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html">GCloud</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html">Flink on Tez <span class="badge">Beta</span></a></li>
+
+                <li class="divider"></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/config.html">Configuration</a></li>
+              </ul>
+            </li>
+
+            <!-- Programming Guides -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/apis" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a>
+              <ul class="dropdown-menu" role="menu">
+                <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html"><strong>Batch: DataSet API</strong></a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html"><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/python.html">Python API <span class="badge">Beta</span></a></li>
+
+                <li class="divider"></li>
+                <li><a href="scala_shell.html">Interactive Scala Shell</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html">Dataset Transformations</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html">Best Practices</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html">Connectors</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html">Examples</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html">Local Execution</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html">Cluster Execution</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html">Command Line Interface</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html">Web Client</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html">Iterations</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html">Java 8</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html">Hadoop Compatability <span class="badge">Beta</span></a></li>
+              </ul>
+            </li>
+
+            <!-- Libraries -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/libs" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a>
+                <ul class="dropdown-menu" role="menu">
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html">Graphs: Spargel</a></li>
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html">Graphs: Gelly <span class="badge">Beta</span></a></li>
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/ml/">Machine Learning <span class="badge">Beta</span></a></li>
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/table.html">Relational: Table <span class="badge">Beta</span></a></li>
+              </ul>
+            </li>
+
+            <!-- Internals -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/internals" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a>
+              <ul class="dropdown-menu" role="menu">
+                <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html">How to Contribute</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html">Coding Guidelines</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html">IDE Setup</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html">Logging</a></li>
+                <li class="divider"></li>
+                <li role="presentation" class="dropdown-header"><strong>Internals</strong></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html">Architecture &amp; Process Model</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html">Type Extraction &amp; Serialization</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html">Jobs &amp; Scheduling</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html">How-To: Add an Operator</a></li>
+              </ul>
+            </li>
+          </ul>
+          <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html">
+            <div class="form-group">
+              <input type="text" class="form-control" name="q" placeholder="Search all pages">
+            </div>
+            <button type="submit" class="btn btn-default">Search</button>
+          </form>
+        </div><!-- /.navbar-collapse -->
+      </div><!-- /.container -->
+    </nav>
+
+
+    
+    
+
+<!--Some of the Latex math notation has been adapted from Apache Spark MLlib's documentation-->
+$$
+\newcommand{\R}{\mathbb{R}}
+\newcommand{\E}{\mathbb{E}}
+\newcommand{\x}{\mathbf{x}}
+\newcommand{\y}{\mathbf{y}}
+\newcommand{\wv}{\mathbf{w}}
+\newcommand{\av}{\mathbf{\alpha}}
+\newcommand{\bv}{\mathbf{b}}
+\newcommand{\N}{\mathbb{N}}
+\newcommand{\id}{\mathbf{I}}
+\newcommand{\ind}{\mathbf{1}}
+\newcommand{\0}{\mathbf{0}}
+\newcommand{\unit}{\mathbf{e}}
+\newcommand{\one}{\mathbf{1}}
+\newcommand{\zero}{\mathbf{0}}
+\newcommand\rfrac[2]{^{#1}\!/_{#2}}
+\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
+$$
+    
+
+    <!-- Main content. -->
+    <div class="container">
+      
+      
+<div class="row">
+  <div class="col-sm-10 col-sm-offset-1">
+    <h1><a href="../ml">FlinkML</a> - Quickstart Guide</h1>
+
+
+
+<ul id="markdown-toc">
+  <li><a href="#introduction" id="markdown-toc-introduction">Introduction</a></li>
+  <li><a href="#linking-with-flinkml" id="markdown-toc-linking-with-flinkml">Linking with FlinkML</a></li>
+  <li><a href="#loading-data" id="markdown-toc-loading-data">Loading data</a></li>
+  <li><a href="#classification" id="markdown-toc-classification">Classification</a></li>
+  <li><a href="#data-pre-processing-and-pipelines" id="markdown-toc-data-pre-processing-and-pipelines">Data pre-processing and pipelines</a></li>
+  <li><a href="#where-to-go-from-here" id="markdown-toc-where-to-go-from-here">Where to go from here</a></li>
+</ul>
+
+<h2 id="introduction">Introduction</h2>
+
+<p>FlinkML is designed to make learning from your data a straight-forward process, abstracting away
+the complexities that usually come with big data learning tasks. In this
+quick-start guide we will show just how easy it is to solve a simple supervised learning problem
+using FlinkML. But first some basics, feel free to skip the next few lines if you’re already
+familiar with Machine Learning (ML).</p>
+
+<p>As defined by Murphy <a href="#murphy">[1]</a> ML deals with detecting patterns in data, and using those
+learned patterns to make predictions about the future. We can categorize most ML algorithms into
+two major categories: Supervised and Unsupervised Learning.</p>
+
+<ul>
+  <li>
+    <p><strong>Supervised Learning</strong> deals with learning a function (mapping) from a set of inputs
+(features) to a set of outputs. The learning is done using a <em>training set</em> of (input,
+output) pairs that we use to approximate the mapping function. Supervised learning problems are
+further divided into classification and regression problems. In classification problems we try to
+predict the <em>class</em> that an example belongs to, for example whether a user is going to click on
+an ad or not. Regression problems one the other hand, are about predicting (real) numerical
+values, often called the dependent variable, for example what the temperature will be tomorrow.</p>
+  </li>
+  <li>
+    <p><strong>Unsupervised Learning</strong> deals with discovering patterns and regularities in the data. An example
+of this would be <em>clustering</em>, where we try to discover groupings of the data from the
+descriptive features. Unsupervised learning can also be used for feature selection, for example
+through <a href="https://en.wikipedia.org/wiki/Principal_component_analysis">principal components analysis</a>.</p>
+  </li>
+</ul>
+
+<h2 id="linking-with-flinkml">Linking with FlinkML</h2>
+
+<p>In order to use FlinkML in your project, first you have to
+<a href="http://ci.apache.org/projects/flink/flink-docs-master/apis/programming_guide.html#linking-with-flink">set up a Flink program</a>.
+Next, you have to add the FlinkML dependency to the <code>pom.xml</code> of your project:</p>
+
+<div class="highlight"><pre><code class="language-xml" data-lang="xml"><span class="nt">&lt;dependency&gt;</span>
+  <span class="nt">&lt;groupId&gt;</span>org.apache.flink<span class="nt">&lt;/groupId&gt;</span>
+  <span class="nt">&lt;artifactId&gt;</span>flink-ml<span class="nt">&lt;/artifactId&gt;</span>
+  <span class="nt">&lt;version&gt;</span>0.9.0<span class="nt">&lt;/version&gt;</span>
+<span class="nt">&lt;/dependency&gt;</span></code></pre></div>
+
+<h2 id="loading-data">Loading data</h2>
+
+<p>To load data to be used with FlinkML we can use the ETL capabilities of Flink, or specialized
+functions for formatted data, such as the LibSVM format. For supervised learning problems it is
+common to use the <code>LabeledVector</code> class to represent the <code>(label, features)</code> examples. A <code>LabeledVector</code>
+object will have a FlinkML <code>Vector</code> member representing the features of the example and a <code>Double</code>
+member which represents the label, which could be the class in a classification problem, or the dependent
+variable for a regression problem.</p>
+
+<p>As an example, we can use Haberman’s Survival Data Set , which you can
+<a href="http://archive.ics.uci.edu/ml/machine-learning-databases/haberman/haberman.data">download from the UCI ML repository</a>.
+This dataset <em>“contains cases from a study conducted on the survival of patients who had undergone
+surgery for breast cancer”</em>. The data comes in a comma-separated file, where the first 3 columns
+are the features and last column is the class, and the 4th column indicates whether the patient
+survived 5 years or longer (label 1), or died within 5 years (label 2). You can check the <a href="https://archive.ics.uci.edu/ml/datasets/Haberman%27s+Survival">UCI
+page</a> for more information on the data.</p>
+
+<p>We can load the data as a <code>DataSet[String]</code> first:</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">import</span> <span class="nn">org.apache.flink.api.scala.ExecutionEnvironment</span>
+
+<span class="k">val</span> <span class="n">env</span> <span class="k">=</span> <span class="nc">ExecutionEnvironment</span><span class="o">.</span><span class="n">getExecutionEnvironment</span>
+
+<span class="k">val</span> <span class="n">survival</span> <span class="k">=</span> <span class="n">env</span><span class="o">.</span><span class="n">readCsvFile</span><span class="o">[(</span><span class="kt">String</span>, <span class="kt">String</span>, <span class="kt">String</span>, <span class="kt">String</span><span class="o">)](</span><span class="s">&quot;/path/to/haberman.data&quot;</span><span class="o">)</span></code></pre></div>
+
+<p>We can now transform the data into a <code>DataSet[LabeledVector]</code>. This will allow us to use the
+dataset with the FlinkML classification algorithms. We know that the 4th element of the dataset
+is the class label, and the rest are features, so we can build <code>LabeledVector</code> elements like this:</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">import</span> <span class="nn">org.apache.flink.ml.common.LabeledVector</span>
+<span class="k">import</span> <span class="nn">org.apache.flink.ml.math.DenseVector</span>
+
+<span class="k">val</span> <span class="n">survivalLV</span> <span class="k">=</span> <span class="n">survival</span>
+  <span class="o">.</span><span class="n">map</span><span class="o">{</span><span class="n">tuple</span> <span class="k">=&gt;</span>
+    <span class="k">val</span> <span class="n">list</span> <span class="k">=</span> <span class="n">tuple</span><span class="o">.</span><span class="n">productIterator</span><span class="o">.</span><span class="n">toList</span>
+    <span class="k">val</span> <span class="n">numList</span> <span class="k">=</span> <span class="n">list</span><span class="o">.</span><span class="n">map</span><span class="o">(</span><span class="k">_</span><span class="o">.</span><span class="n">asInstanceOf</span><span class="o">[</span><span class="kt">String</span><span class="o">].</span><span class="n">toDouble</span><span class="o">)</span>
+    <span class="nc">LabeledVector</span><span class="o">(</span><span class="n">numList</span><span class="o">(</span><span class="mi">3</span><span class="o">),</span> <span class="nc">DenseVector</span><span class="o">(</span><span class="n">numList</span><span class="o">.</span><span class="n">take</span><span class="o">(</span><span class="mi">3</span><span class="o">).</span><span class="n">toArray</span><span class="o">))</span>
+  <span class="o">}</span></code></pre></div>
+
+<p>We can then use this data to train a learner. We will however use another dataset to exemplify
+building a learner; that will allow us to show how we can import other dataset formats.</p>
+
+<p><strong>LibSVM files</strong></p>
+
+<p>A common format for ML datasets is the LibSVM format and a number of datasets using that format can be
+found <a href="http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/">in the LibSVM datasets website</a>. FlinkML provides utilities for loading
+datasets using the LibSVM format through the <code>readLibSVM</code> function available through the <code>MLUtils</code>
+object.
+You can also save datasets in the LibSVM format using the <code>writeLibSVM</code> function.
+Let’s import the svmguide1 dataset. You can download the
+<a href="http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/svmguide1">training set here</a>
+and the <a href="http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/svmguide1.t">test set here</a>.
+This is an astroparticle binary classification dataset, used by Hsu et al. <a href="#hsu">[3]</a> in their 
+practical Support Vector Machine (SVM) guide. It contains 4 numerical features, and the class label.</p>
+
+<p>We can simply import the dataset then using:</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">import</span> <span class="nn">org.apache.flink.ml.MLUtils</span>
+
+<span class="k">val</span> <span class="n">astroTrain</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="nc">MLUtils</span><span class="o">.</span><span class="n">readLibSVM</span><span class="o">(</span><span class="s">&quot;/path/to/svmguide1&quot;</span><span class="o">)</span>
+<span class="k">val</span> <span class="n">astroTest</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="nc">MLUtils</span><span class="o">.</span><span class="n">readLibSVM</span><span class="o">(</span><span class="s">&quot;/path/to/svmguide1.t&quot;</span><span class="o">)</span></code></pre></div>
+
+<p>This gives us two <code>DataSet[LabeledVector]</code> objects that we will use in the following section to
+create a classifier.</p>
+
+<h2 id="classification">Classification</h2>
+
+<p>Once we have imported the dataset we can train a <code>Predictor</code> such as a linear SVM classifier.
+We can set a number of parameters for the classifier. Here we set the <code>Blocks</code> parameter,
+which is used to split the input by the underlying CoCoA algorithm <a href="#jaggi">[2]</a> uses. The 
+regularization parameter determines the amount of $l_2$ regularization applied, which is used
+to avoid overfitting. The step size determines the contribution of the weight vector updates to
+the next weight vector value. This parameter sets the initial step size.</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">import</span> <span class="nn">org.apache.flink.ml.classification.SVM</span>
+
+<span class="k">val</span> <span class="n">svm</span> <span class="k">=</span> <span class="nc">SVM</span><span class="o">()</span>
+  <span class="o">.</span><span class="n">setBlocks</span><span class="o">(</span><span class="n">env</span><span class="o">.</span><span class="n">getParallelism</span><span class="o">)</span>
+  <span class="o">.</span><span class="n">setIterations</span><span class="o">(</span><span class="mi">100</span><span class="o">)</span>
+  <span class="o">.</span><span class="n">setRegularization</span><span class="o">(</span><span class="mf">0.001</span><span class="o">)</span>
+  <span class="o">.</span><span class="n">setStepsize</span><span class="o">(</span><span class="mf">0.1</span><span class="o">)</span>
+  <span class="o">.</span><span class="n">setSeed</span><span class="o">(</span><span class="mi">42</span><span class="o">)</span>
+
+<span class="n">svm</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">astroTrain</span><span class="o">)</span></code></pre></div>
+
+<p>We can now make predictions on the test set.</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">val</span> <span class="n">predictionPairs</span> <span class="k">=</span> <span class="n">svm</span><span class="o">.</span><span class="n">predict</span><span class="o">(</span><span class="n">astroTest</span><span class="o">)</span></code></pre></div>
+
+<p>Next we will see how we can pre-process our data, and use the ML pipelines capabilities of FlinkML.</p>
+
+<h2 id="data-pre-processing-and-pipelines">Data pre-processing and pipelines</h2>
+
+<p>A pre-processing step that is often encouraged <a href="#hsu">[3]</a> when using SVM classification is scaling
+the input features to the [0, 1] range, in order to avoid features with extreme values
+dominating the rest.
+FlinkML has a number of <code>Transformers</code> such as <code>MinMaxScaler</code> that are used to pre-process data,
+and a key feature is the ability to chain <code>Transformers</code> and <code>Predictors</code> together. This allows
+us to run the same pipeline of transformations and make predictions on the train and test data in
+a straight-forward and type-safe manner. You can read more on the pipeline system of FlinkML
+<a href="pipelines.html">in the pipelines documentation</a>.</p>
+
+<p>Let us first create a normalizing transformer for the features in our dataset, and chain it to a
+new SVM classifier.</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">import</span> <span class="nn">org.apache.flink.ml.preprocessing.MinMaxScaler</span>
+
+<span class="k">val</span> <span class="n">scaler</span> <span class="k">=</span> <span class="nc">MinMaxScaler</span><span class="o">()</span>
+
+<span class="k">val</span> <span class="n">scaledSVM</span> <span class="k">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">chainPredictor</span><span class="o">(</span><span class="n">svm</span><span class="o">)</span></code></pre></div>
+
+<p>We can now use our newly created pipeline to make predictions on the test set.
+First we call fit again, to train the scaler and the SVM classifier.
+The data of the test set will then be automatically scaled before being passed on to the SVM to
+make predictions.</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="n">scaledSVM</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">astroTrain</span><span class="o">)</span>
+
+<span class="k">val</span> <span class="n">predictionPairsScaled</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[(</span><span class="kt">Double</span>, <span class="kt">Double</span><span class="o">)]</span> <span class="k">=</span> <span class="n">scaledSVM</span><span class="o">.</span><span class="n">predict</span><span class="o">(</span><span class="n">astroTest</span><span class="o">)</span></code></pre></div>
+
+<p>The scaled inputs should give us better prediction performance.
+The result of the prediction on <code>LabeledVector</code>s is a data set of tuples where the first entry denotes the true label value and the second entry is the predicted label value.</p>
+
+<h2 id="where-to-go-from-here">Where to go from here</h2>
+
+<p>This quickstart guide can act as an introduction to the basic concepts of FlinkML, but there’s a lot
+more you can do.
+We recommend going through the <a href="index.html">FlinkML documentation</a>, and trying out the different
+algorithms.
+A very good way to get started is to play around with interesting datasets from the UCI ML
+repository and the LibSVM datasets.
+Tackling an interesting problem from a website like <a href="https://www.kaggle.com">Kaggle</a> or
+<a href="http://www.drivendata.org/">DrivenData</a> is also a great way to learn by competing with other
+data scientists.
+If you would like to contribute some new algorithms take a look at our
+<a href="contribution_guide.html">contribution guide</a>.</p>
+
+<p><strong>References</strong></p>
+
+<p><a name="murphy"></a>[1] Murphy, Kevin P. <em>Machine learning: a probabilistic perspective.</em> MIT 
+press, 2012.</p>
+
+<p><a name="jaggi"></a>[2] Jaggi, Martin, et al. <em>Communication-efficient distributed dual 
+coordinate ascent.</em> Advances in Neural Information Processing Systems. 2014.</p>
+
+<p><a name="hsu"></a>[3] Hsu, Chih-Wei, Chih-Chung Chang, and Chih-Jen Lin.
+ <em>A practical guide to support vector classification.</em> 2003.</p>
+
+  </div>
+
+  <div class="col-sm-10 col-sm-offset-1">
+    <!-- Disqus thread and some vertical offset -->
+    <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div>
+  </div>
+</div>
+
+    </div><!-- /.container -->
+
+    <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
+    <!-- Include all compiled plugins (below), or include individual files as needed -->
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
+    <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js"></script>
+
+    <!-- Google Analytics -->
+    <script>
+      (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+      })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+      ga('create', 'UA-52545728-1', 'auto');
+      ga('send', 'pageview');
+    </script>
+
+    <!-- Disqus -->
+    <script type="text/javascript">
+    var disqus_shortname = 'stratosphere-eu';
+    (function() {
+        var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
+        dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
+        (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
+    })();
+</script>
+  </body>
+</html>

http://git-wip-us.apache.org/repos/asf/flink-web/blob/396616d4/content/docs/0.9/libs/ml/standard_scaler.html
----------------------------------------------------------------------
diff --git a/content/docs/0.9/libs/ml/standard_scaler.html b/content/docs/0.9/libs/ml/standard_scaler.html
new file mode 100644
index 0000000..9f84bfa
--- /dev/null
+++ b/content/docs/0.9/libs/ml/standard_scaler.html
@@ -0,0 +1,340 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE html>
+
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+    
+    <title>Apache Flink 0.9.0 Documentation: FlinkML - Standard Scaler</title>
+    
+    <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+    <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+
+    <!-- Bootstrap -->
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
+    <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css">
+    <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css">
+    <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css">
+    
+    <script type="text/x-mathjax-config">
+        MathJax.Hub.Config({
+        tex2jax: {
+          inlineMath: [['$','$'], ['\\(','\\)']] },
+        TeX: {
+          equationNumbers: { autoNumber: "AMS" } }
+        });
+    </script>
+    <script type="text/javascript"
+            src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
+    </script>
+    
+    <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+    <!--[if lt IE 9]>
+      <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
+      <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+    <![endif]-->
+  </head>
+  <body>
+    
+    
+
+
+
+
+    <!-- Top navbar. -->
+    <nav class="navbar navbar-default navbar-fixed-top">
+      <div class="container">
+        <!-- The logo. -->
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <div class="navbar-logo">
+            <a href="http://flink.apache.org"><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"></a>
+          </div>
+        </div><!-- /.navbar-header -->
+
+        <!-- The navigation links. -->
+        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+          <ul class="nav navbar-nav">
+            <li><a href="http://flink.apache.org/docs/0.9/index.html">Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li>
+
+            <!-- Setup -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/setup" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a>
+              <ul class="dropdown-menu" role="menu">
+                <li><a href="http://flink.apache.org/docs/0.9/setup/building.html">Get Flink 0.9-SNAPSHOT</a></li>
+
+                <li class="divider"></li>
+                <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html" class="active">Local</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html">Cluster (Standalone)</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html">YARN</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html">GCloud</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html">Flink on Tez <span class="badge">Beta</span></a></li>
+
+                <li class="divider"></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/config.html">Configuration</a></li>
+              </ul>
+            </li>
+
+            <!-- Programming Guides -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/apis" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a>
+              <ul class="dropdown-menu" role="menu">
+                <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html"><strong>Batch: DataSet API</strong></a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html"><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/python.html">Python API <span class="badge">Beta</span></a></li>
+
+                <li class="divider"></li>
+                <li><a href="scala_shell.html">Interactive Scala Shell</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html">Dataset Transformations</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html">Best Practices</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html">Connectors</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html">Examples</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html">Local Execution</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html">Cluster Execution</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html">Command Line Interface</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html">Web Client</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html">Iterations</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html">Java 8</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html">Hadoop Compatability <span class="badge">Beta</span></a></li>
+              </ul>
+            </li>
+
+            <!-- Libraries -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/libs" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a>
+                <ul class="dropdown-menu" role="menu">
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html">Graphs: Spargel</a></li>
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html">Graphs: Gelly <span class="badge">Beta</span></a></li>
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/ml/">Machine Learning <span class="badge">Beta</span></a></li>
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/table.html">Relational: Table <span class="badge">Beta</span></a></li>
+              </ul>
+            </li>
+
+            <!-- Internals -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/internals" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a>
+              <ul class="dropdown-menu" role="menu">
+                <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html">How to Contribute</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html">Coding Guidelines</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html">IDE Setup</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html">Logging</a></li>
+                <li class="divider"></li>
+                <li role="presentation" class="dropdown-header"><strong>Internals</strong></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html">Architecture &amp; Process Model</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html">Type Extraction &amp; Serialization</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html">Jobs &amp; Scheduling</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html">How-To: Add an Operator</a></li>
+              </ul>
+            </li>
+          </ul>
+          <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html">
+            <div class="form-group">
+              <input type="text" class="form-control" name="q" placeholder="Search all pages">
+            </div>
+            <button type="submit" class="btn btn-default">Search</button>
+          </form>
+        </div><!-- /.navbar-collapse -->
+      </div><!-- /.container -->
+    </nav>
+
+
+    
+    
+
+<!--Some of the Latex math notation has been adapted from Apache Spark MLlib's documentation-->
+$$
+\newcommand{\R}{\mathbb{R}}
+\newcommand{\E}{\mathbb{E}}
+\newcommand{\x}{\mathbf{x}}
+\newcommand{\y}{\mathbf{y}}
+\newcommand{\wv}{\mathbf{w}}
+\newcommand{\av}{\mathbf{\alpha}}
+\newcommand{\bv}{\mathbf{b}}
+\newcommand{\N}{\mathbb{N}}
+\newcommand{\id}{\mathbf{I}}
+\newcommand{\ind}{\mathbf{1}}
+\newcommand{\0}{\mathbf{0}}
+\newcommand{\unit}{\mathbf{e}}
+\newcommand{\one}{\mathbf{1}}
+\newcommand{\zero}{\mathbf{0}}
+\newcommand\rfrac[2]{^{#1}\!/_{#2}}
+\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
+$$
+    
+
+    <!-- Main content. -->
+    <div class="container">
+      
+      
+<div class="row">
+  <div class="col-sm-10 col-sm-offset-1">
+    <h1><a href="../ml">FlinkML</a> - Standard Scaler</h1>
+
+
+
+<ul id="markdown-toc">
+  <li><a href="#description" id="markdown-toc-description">Description</a></li>
+  <li><a href="#operations" id="markdown-toc-operations">Operations</a>    <ul>
+      <li><a href="#fit" id="markdown-toc-fit">Fit</a></li>
+      <li><a href="#transform" id="markdown-toc-transform">Transform</a></li>
+    </ul>
+  </li>
+  <li><a href="#parameters" id="markdown-toc-parameters">Parameters</a></li>
+  <li><a href="#examples" id="markdown-toc-examples">Examples</a></li>
+</ul>
+
+<h2 id="description">Description</h2>
+
+<p>The standard scaler scales the given data set, so that all features will have a user specified mean and variance. 
+ In case the user does not provide a specific mean and standard deviation, the standard scaler transforms the features of the input data set to have mean equal to 0 and standard deviation equal to 1.
+ Given a set of input data $x_1, x_2,… x_n$, with mean:</p>
+
+<script type="math/tex; mode=display">\bar{x} = \frac{1}{n}\sum_{i=1}^{n}x_{i}</script>
+
+<p>and standard deviation:</p>
+
+<script type="math/tex; mode=display">\sigma_{x}=\sqrt{ \frac{1}{n} \sum_{i=1}^{n}(x_{i}-\bar{x})^{2}}</script>
+
+<p>The scaled data set $z_1, z_2,…,z_n$ will be:</p>
+
+<script type="math/tex; mode=display">z_{i}= std \left (\frac{x_{i} - \bar{x}  }{\sigma_{x}}\right ) + mean</script>
+
+<p>where $\textit{std}$ and $\textit{mean}$ are the user specified values for the standard deviation and mean.</p>
+
+<h2 id="operations">Operations</h2>
+
+<p><code>StandardScaler</code> is a <code>Transformer</code>.
+As such, it supports the <code>fit</code> and <code>transform</code> operation.</p>
+
+<h3 id="fit">Fit</h3>
+
+<p>StandardScaler is trained on all subtypes of <code>Vector</code> or <code>LabeledVector</code>:</p>
+
+<ul>
+  <li><code>fit[T &lt;: Vector]: DataSet[T] =&gt; Unit</code></li>
+  <li><code>fit: DataSet[LabeledVector] =&gt; Unit</code></li>
+</ul>
+
+<h3 id="transform">Transform</h3>
+
+<p>StandardScaler transforms all subtypes of <code>Vector</code> or <code>LabeledVector</code> into the respective type:</p>
+
+<ul>
+  <li><code>transform[T &lt;: Vector]: DataSet[T] =&gt; DataSet[T]</code></li>
+  <li><code>transform: DataSet[LabeledVector] =&gt; DataSet[LabeledVector]</code></li>
+</ul>
+
+<h2 id="parameters">Parameters</h2>
+
+<p>The standard scaler implementation can be controlled by the following two parameters:</p>
+
+<table class="table table-bordered">
+  <thead>
+    <tr>
+      <th class="text-left" style="width: 20%">Parameters</th>
+      <th class="text-center">Description</th>
+    </tr>
+  </thead>
+
+  <tbody>
+    <tr>
+      <td><strong>Mean</strong></td>
+      <td>
+        <p>
+          The mean of the scaled data set. (Default value: <strong>0.0</strong>)
+        </p>
+      </td>
+    </tr>
+    <tr>
+      <td><strong>Std</strong></td>
+      <td>
+        <p>
+          The standard deviation of the scaled data set. (Default value: <strong>1.0</strong>)
+        </p>
+      </td>
+    </tr>
+  </tbody>
+</table>
+
+<h2 id="examples">Examples</h2>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="c1">// Create standard scaler transformer</span>
+<span class="k">val</span> <span class="n">scaler</span> <span class="k">=</span> <span class="nc">StandardScaler</span><span class="o">()</span>
+<span class="o">.</span><span class="n">setMean</span><span class="o">(</span><span class="mf">10.0</span><span class="o">)</span>
+<span class="o">.</span><span class="n">setStd</span><span class="o">(</span><span class="mf">2.0</span><span class="o">)</span>
+
+<span class="c1">// Obtain data set to be scaled</span>
+<span class="k">val</span> <span class="n">dataSet</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">Vector</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span>
+
+<span class="c1">// Learn the mean and standard deviation of the training data</span>
+<span class="n">scaler</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">dataSet</span><span class="o">)</span>
+
+<span class="c1">// Scale the provided data set to have mean=10.0 and std=2.0</span>
+<span class="k">val</span> <span class="n">scaledDS</span> <span class="k">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="o">(</span><span class="n">dataSet</span><span class="o">)</span></code></pre></div>
+
+
+  </div>
+
+  <div class="col-sm-10 col-sm-offset-1">
+    <!-- Disqus thread and some vertical offset -->
+    <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div>
+  </div>
+</div>
+
+    </div><!-- /.container -->
+
+    <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
+    <!-- Include all compiled plugins (below), or include individual files as needed -->
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
+    <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js"></script>
+
+    <!-- Google Analytics -->
+    <script>
+      (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+      })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+      ga('create', 'UA-52545728-1', 'auto');
+      ga('send', 'pageview');
+    </script>
+
+    <!-- Disqus -->
+    <script type="text/javascript">
+    var disqus_shortname = 'stratosphere-eu';
+    (function() {
+        var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
+        dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
+        (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
+    })();
+</script>
+  </body>
+</html>

http://git-wip-us.apache.org/repos/asf/flink-web/blob/396616d4/content/docs/0.9/libs/ml/svm.html
----------------------------------------------------------------------
diff --git a/content/docs/0.9/libs/ml/svm.html b/content/docs/0.9/libs/ml/svm.html
new file mode 100644
index 0000000..00184c4
--- /dev/null
+++ b/content/docs/0.9/libs/ml/svm.html
@@ -0,0 +1,419 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE html>
+
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+    
+    <title>Apache Flink 0.9.0 Documentation: FlinkML - SVM using CoCoA</title>
+    
+    <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+    <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+
+    <!-- Bootstrap -->
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
+    <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css">
+    <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css">
+    <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css">
+    
+    <script type="text/x-mathjax-config">
+        MathJax.Hub.Config({
+        tex2jax: {
+          inlineMath: [['$','$'], ['\\(','\\)']] },
+        TeX: {
+          equationNumbers: { autoNumber: "AMS" } }
+        });
+    </script>
+    <script type="text/javascript"
+            src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
+    </script>
+    
+    <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+    <!--[if lt IE 9]>
+      <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
+      <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+    <![endif]-->
+  </head>
+  <body>
+    
+    
+
+
+
+
+    <!-- Top navbar. -->
+    <nav class="navbar navbar-default navbar-fixed-top">
+      <div class="container">
+        <!-- The logo. -->
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <div class="navbar-logo">
+            <a href="http://flink.apache.org"><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"></a>
+          </div>
+        </div><!-- /.navbar-header -->
+
+        <!-- The navigation links. -->
+        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+          <ul class="nav navbar-nav">
+            <li><a href="http://flink.apache.org/docs/0.9/index.html">Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li>
+
+            <!-- Setup -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/setup" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a>
+              <ul class="dropdown-menu" role="menu">
+                <li><a href="http://flink.apache.org/docs/0.9/setup/building.html">Get Flink 0.9-SNAPSHOT</a></li>
+
+                <li class="divider"></li>
+                <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html" class="active">Local</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html">Cluster (Standalone)</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html">YARN</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html">GCloud</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html">Flink on Tez <span class="badge">Beta</span></a></li>
+
+                <li class="divider"></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/config.html">Configuration</a></li>
+              </ul>
+            </li>
+
+            <!-- Programming Guides -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/apis" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a>
+              <ul class="dropdown-menu" role="menu">
+                <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html"><strong>Batch: DataSet API</strong></a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html"><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/python.html">Python API <span class="badge">Beta</span></a></li>
+
+                <li class="divider"></li>
+                <li><a href="scala_shell.html">Interactive Scala Shell</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html">Dataset Transformations</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html">Best Practices</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html">Connectors</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html">Examples</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html">Local Execution</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html">Cluster Execution</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html">Command Line Interface</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html">Web Client</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html">Iterations</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html">Java 8</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html">Hadoop Compatability <span class="badge">Beta</span></a></li>
+              </ul>
+            </li>
+
+            <!-- Libraries -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/libs" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a>
+                <ul class="dropdown-menu" role="menu">
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html">Graphs: Spargel</a></li>
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html">Graphs: Gelly <span class="badge">Beta</span></a></li>
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/ml/">Machine Learning <span class="badge">Beta</span></a></li>
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/table.html">Relational: Table <span class="badge">Beta</span></a></li>
+              </ul>
+            </li>
+
+            <!-- Internals -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/internals" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a>
+              <ul class="dropdown-menu" role="menu">
+                <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html">How to Contribute</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html">Coding Guidelines</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html">IDE Setup</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html">Logging</a></li>
+                <li class="divider"></li>
+                <li role="presentation" class="dropdown-header"><strong>Internals</strong></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html">Architecture &amp; Process Model</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html">Type Extraction &amp; Serialization</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html">Jobs &amp; Scheduling</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html">How-To: Add an Operator</a></li>
+              </ul>
+            </li>
+          </ul>
+          <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html">
+            <div class="form-group">
+              <input type="text" class="form-control" name="q" placeholder="Search all pages">
+            </div>
+            <button type="submit" class="btn btn-default">Search</button>
+          </form>
+        </div><!-- /.navbar-collapse -->
+      </div><!-- /.container -->
+    </nav>
+
+
+    
+    
+
+<!--Some of the Latex math notation has been adapted from Apache Spark MLlib's documentation-->
+$$
+\newcommand{\R}{\mathbb{R}}
+\newcommand{\E}{\mathbb{E}}
+\newcommand{\x}{\mathbf{x}}
+\newcommand{\y}{\mathbf{y}}
+\newcommand{\wv}{\mathbf{w}}
+\newcommand{\av}{\mathbf{\alpha}}
+\newcommand{\bv}{\mathbf{b}}
+\newcommand{\N}{\mathbb{N}}
+\newcommand{\id}{\mathbf{I}}
+\newcommand{\ind}{\mathbf{1}}
+\newcommand{\0}{\mathbf{0}}
+\newcommand{\unit}{\mathbf{e}}
+\newcommand{\one}{\mathbf{1}}
+\newcommand{\zero}{\mathbf{0}}
+\newcommand\rfrac[2]{^{#1}\!/_{#2}}
+\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
+$$
+    
+
+    <!-- Main content. -->
+    <div class="container">
+      
+      
+<div class="row">
+  <div class="col-sm-10 col-sm-offset-1">
+    <h1><a href="../ml">FlinkML</a> - SVM using CoCoA</h1>
+
+
+
+<ul id="markdown-toc">
+  <li><a href="#description" id="markdown-toc-description">Description</a></li>
+  <li><a href="#operations" id="markdown-toc-operations">Operations</a>    <ul>
+      <li><a href="#fit" id="markdown-toc-fit">Fit</a></li>
+      <li><a href="#predict" id="markdown-toc-predict">Predict</a></li>
+    </ul>
+  </li>
+  <li><a href="#parameters" id="markdown-toc-parameters">Parameters</a></li>
+  <li><a href="#examples" id="markdown-toc-examples">Examples</a></li>
+</ul>
+
+<h2 id="description">Description</h2>
+
+<p>Implements an SVM with soft-margin using the communication-efficient distributed dual coordinate 
+ascent algorithm with hinge-loss function. 
+The algorithm solves the following minimization problem:</p>
+
+<script type="math/tex; mode=display">\min_{\mathbf{w} \in \mathbb{R}^d} \frac{\lambda}{2} \left\lVert \mathbf{w} \right\rVert^2 + \frac{1}{n} \sum_{i=1}^n l_{i}\left(\mathbf{w}^T\mathbf{x}_i\right)</script>
+
+<p>with $\mathbf{w}$ being the weight vector, $\lambda$ being the regularization constant, 
+<script type="math/tex">\mathbf{x}_i \in \mathbb{R}^d</script> being the data points and <script type="math/tex">l_{i}</script> being the convex loss 
+functions, which can also depend on the labels <script type="math/tex">y_{i} \in \mathbb{R}</script>.
+In the current implementation the regularizer is the $\ell_2$-norm and the loss functions are the hinge-loss functions:</p>
+
+<script type="math/tex; mode=display">l_{i} = \max\left(0, 1 - y_{i} \mathbf{w}^T\mathbf{x}_i \right)</script>
+
+<p>With these choices, the problem definition is equivalent to a SVM with soft-margin.
+Thus, the algorithm allows us to train a SVM with soft-margin.</p>
+
+<p>The minimization problem is solved by applying stochastic dual coordinate ascent (SDCA).
+In order to make the algorithm efficient in a distributed setting, the CoCoA algorithm calculates 
+several iterations of SDCA locally on a data block before merging the local updates into a
+valid global state.
+This state is redistributed to the different data partitions where the next round of local SDCA 
+iterations is then executed.
+The number of outer iterations and local SDCA iterations control the overall network costs, because 
+there is only network communication required for each outer iteration.
+The local SDCA iterations are embarrassingly parallel once the individual data partitions have been 
+distributed across the cluster.</p>
+
+<p>The implementation of this algorithm is based on the work of 
+<a href="http://arxiv.org/abs/1409.1458">Jaggi et al.</a></p>
+
+<h2 id="operations">Operations</h2>
+
+<p><code>SVM</code> is a <code>Predictor</code>.
+As such, it supports the <code>fit</code> and <code>predict</code> operation.</p>
+
+<h3 id="fit">Fit</h3>
+
+<p>SVM is trained given a set of <code>LabeledVector</code>:</p>
+
+<ul>
+  <li><code>fit: DataSet[LabeledVector] =&gt; Unit</code></li>
+</ul>
+
+<h3 id="predict">Predict</h3>
+
+<p>SVM predicts for all subtypes of <code>Vector</code> the corresponding class label:</p>
+
+<ul>
+  <li><code>predict[T &lt;: Vector]: DataSet[T] =&gt; DataSet[LabeledVector]</code></li>
+</ul>
+
+<p>If we call predict with a <code>DataSet[LabeledVector]</code>, we make a prediction on the class label
+for each example, and return a <code>DataSet[(Double, Double)]</code>. In each tuple the first element
+is the true value, as was provided from the input <code>DataSet[LabeledVector]</code> and the second element
+is the predicted value. You can then use these <code>(truth, prediction)</code> tuples to evaluate
+the algorithm’s performance.</p>
+
+<ul>
+  <li><code>predict: DataSet[LabeledVector] =&gt; DataSet[(Double, Double)]</code></li>
+</ul>
+
+<h2 id="parameters">Parameters</h2>
+
+<p>The SVM implementation can be controlled by the following parameters:</p>
+
+<table class="table table-bordered">
+    <thead>
+      <tr>
+        <th class="text-left" style="width: 20%">Parameters</th>
+        <th class="text-center">Description</th>
+      </tr>
+    </thead>
+
+    <tbody>
+      <tr>
+        <td><strong>Blocks</strong></td>
+        <td>
+          <p>
+            Sets the number of blocks into which the input data will be split. 
+            On each block the local stochastic dual coordinate ascent method is executed. 
+            This number should be set at least to the degree of parallelism. 
+            If no value is specified, then the parallelism of the input DataSet is used as the number of blocks. 
+            (Default value: <strong>None</strong>)
+          </p>
+        </td>
+      </tr>
+      <tr>
+        <td><strong>Iterations</strong></td>
+        <td>
+          <p>
+            Defines the maximum number of iterations of the outer loop method. 
+            In other words, it defines how often the SDCA method is applied to the blocked data. 
+            After each iteration, the locally computed weight vector updates have to be reduced to update the global weight vector value.
+            The new weight vector is broadcast to all SDCA tasks at the beginning of each iteration.
+            (Default value: <strong>10</strong>)
+          </p>
+        </td>
+      </tr>
+      <tr>
+        <td><strong>LocalIterations</strong></td>
+        <td>
+          <p>
+            Defines the maximum number of SDCA iterations. 
+            In other words, it defines how many data points are drawn from each local data block to calculate the stochastic dual coordinate ascent.
+            (Default value: <strong>10</strong>)
+          </p>
+        </td>
+      </tr>
+      <tr>
+        <td><strong>Regularization</strong></td>
+        <td>
+          <p>
+            Defines the regularization constant of the SVM algorithm. 
+            The higher the value, the smaller will the 2-norm of the weight vector be. 
+            In case of a SVM with hinge loss this means that the SVM margin will be wider even though it might contain some false classifications.
+            (Default value: <strong>1.0</strong>)
+          </p>
+        </td>
+      </tr>
+      <tr>
+        <td><strong>Stepsize</strong></td>
+        <td>
+          <p>
+            Defines the initial step size for the updates of the weight vector. 
+            The larger the step size is, the larger will be the contribution of the weight vector updates to the next weight vector value. 
+            The effective scaling of the updates is $\frac{stepsize}{blocks}$.
+            This value has to be tuned in case that the algorithm becomes unstable. 
+            (Default value: <strong>1.0</strong>)
+          </p>
+        </td>
+      </tr>
+      <tr>
+        <td><strong>Seed</strong></td>
+        <td>
+          <p>
+            Defines the seed to initialize the random number generator. 
+            The seed directly controls which data points are chosen for the SDCA method. 
+            (Default value: <strong>0</strong>)
+          </p>
+        </td>
+      </tr>
+    </tbody>
+  </table>
+
+<h2 id="examples">Examples</h2>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="c1">// Read the training data set, from a LibSVM formatted file</span>
+<span class="k">val</span> <span class="n">trainingDS</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="n">env</span><span class="o">.</span><span class="n">readLibSVM</span><span class="o">(</span><span class="n">pathToTrainingFile</span><span class="o">)</span>
+
+<span class="c1">// Create the SVM learner</span>
+<span class="k">val</span> <span class="n">svm</span> <span class="k">=</span> <span class="nc">SVM</span><span class="o">()</span>
+<span class="o">.</span><span class="n">setBlocks</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
+<span class="o">.</span><span class="n">setIterations</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
+<span class="o">.</span><span class="n">setLocalIterations</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
+<span class="o">.</span><span class="n">setRegularization</span><span class="o">(</span><span class="mf">0.5</span><span class="o">)</span>
+<span class="o">.</span><span class="n">setStepsize</span><span class="o">(</span><span class="mf">0.5</span><span class="o">)</span>
+
+<span class="c1">// Learn the SVM model</span>
+<span class="n">svm</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">trainingDS</span><span class="o">)</span>
+
+<span class="c1">// Read the testing data set</span>
+<span class="k">val</span> <span class="n">testingDS</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">Vector</span><span class="o">]</span> <span class="k">=</span> <span class="n">env</span><span class="o">.</span><span class="n">readVectorFile</span><span class="o">(</span><span class="n">pathToTestingFile</span><span class="o">)</span>
+
+<span class="c1">// Calculate the predictions for the testing data set</span>
+<span class="k">val</span> <span class="n">predictionDS</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="n">svm</span><span class="o">.</span><span class="n">predict</span><span class="o">(</span><span class="n">testingDS</span><span class="o">)</span></code></pre></div>
+
+
+  </div>
+
+  <div class="col-sm-10 col-sm-offset-1">
+    <!-- Disqus thread and some vertical offset -->
+    <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div>
+  </div>
+</div>
+
+    </div><!-- /.container -->
+
+    <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
+    <!-- Include all compiled plugins (below), or include individual files as needed -->
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
+    <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js"></script>
+
+    <!-- Google Analytics -->
+    <script>
+      (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+      })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+      ga('create', 'UA-52545728-1', 'auto');
+      ga('send', 'pageview');
+    </script>
+
+    <!-- Disqus -->
+    <script type="text/javascript">
+    var disqus_shortname = 'stratosphere-eu';
+    (function() {
+        var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
+        dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
+        (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
+    })();
+</script>
+  </body>
+</html>

http://git-wip-us.apache.org/repos/asf/flink-web/blob/396616d4/content/docs/0.9/libs/ml/vision_roadmap.html
----------------------------------------------------------------------
diff --git a/content/docs/0.9/libs/ml/vision_roadmap.html b/content/docs/0.9/libs/ml/vision_roadmap.html
new file mode 100644
index 0000000..5793d68
--- /dev/null
+++ b/content/docs/0.9/libs/ml/vision_roadmap.html
@@ -0,0 +1,313 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE html>
+
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+    
+    <title>Apache Flink 0.9.0 Documentation: FlinkML - Vision and Roadmap</title>
+    
+    <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+    <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+
+    <!-- Bootstrap -->
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
+    <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css">
+    <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css">
+    <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css">
+    
+    <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+    <!--[if lt IE 9]>
+      <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
+      <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+    <![endif]-->
+  </head>
+  <body>
+    
+    
+
+
+
+
+    <!-- Top navbar. -->
+    <nav class="navbar navbar-default navbar-fixed-top">
+      <div class="container">
+        <!-- The logo. -->
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <div class="navbar-logo">
+            <a href="http://flink.apache.org"><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"></a>
+          </div>
+        </div><!-- /.navbar-header -->
+
+        <!-- The navigation links. -->
+        <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+          <ul class="nav navbar-nav">
+            <li><a href="http://flink.apache.org/docs/0.9/index.html">Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li>
+
+            <!-- Setup -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/setup" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a>
+              <ul class="dropdown-menu" role="menu">
+                <li><a href="http://flink.apache.org/docs/0.9/setup/building.html">Get Flink 0.9-SNAPSHOT</a></li>
+
+                <li class="divider"></li>
+                <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html" class="active">Local</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html">Cluster (Standalone)</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html">YARN</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html">GCloud</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html">Flink on Tez <span class="badge">Beta</span></a></li>
+
+                <li class="divider"></li>
+                <li><a href="http://flink.apache.org/docs/0.9/setup/config.html">Configuration</a></li>
+              </ul>
+            </li>
+
+            <!-- Programming Guides -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/apis" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a>
+              <ul class="dropdown-menu" role="menu">
+                <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html"><strong>Batch: DataSet API</strong></a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html"><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/python.html">Python API <span class="badge">Beta</span></a></li>
+
+                <li class="divider"></li>
+                <li><a href="scala_shell.html">Interactive Scala Shell</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html">Dataset Transformations</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html">Best Practices</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html">Connectors</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html">Examples</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html">Local Execution</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html">Cluster Execution</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html">Command Line Interface</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html">Web Client</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html">Iterations</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html">Java 8</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html">Hadoop Compatability <span class="badge">Beta</span></a></li>
+              </ul>
+            </li>
+
+            <!-- Libraries -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/libs" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a>
+                <ul class="dropdown-menu" role="menu">
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html">Graphs: Spargel</a></li>
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html">Graphs: Gelly <span class="badge">Beta</span></a></li>
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/ml/">Machine Learning <span class="badge">Beta</span></a></li>
+                  <li><a href="http://flink.apache.org/docs/0.9/libs/table.html">Relational: Table <span class="badge">Beta</span></a></li>
+              </ul>
+            </li>
+
+            <!-- Internals -->
+            <li class="dropdown">
+              <a href="http://flink.apache.org/docs/0.9/internals" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a>
+              <ul class="dropdown-menu" role="menu">
+                <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html">How to Contribute</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html">Coding Guidelines</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html">IDE Setup</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html">Logging</a></li>
+                <li class="divider"></li>
+                <li role="presentation" class="dropdown-header"><strong>Internals</strong></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html">Architecture &amp; Process Model</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html">Type Extraction &amp; Serialization</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html">Jobs &amp; Scheduling</a></li>
+                <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html">How-To: Add an Operator</a></li>
+              </ul>
+            </li>
+          </ul>
+          <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html">
+            <div class="form-group">
+              <input type="text" class="form-control" name="q" placeholder="Search all pages">
+            </div>
+            <button type="submit" class="btn btn-default">Search</button>
+          </form>
+        </div><!-- /.navbar-collapse -->
+      </div><!-- /.container -->
+    </nav>
+
+
+    
+
+    <!-- Main content. -->
+    <div class="container">
+      
+      
+<div class="row">
+  <div class="col-sm-10 col-sm-offset-1">
+    <h1><a href="../ml">FlinkML</a> - Vision and Roadmap</h1>
+
+
+
+<ul id="markdown-toc">
+  <li><a href="#vision" id="markdown-toc-vision">Vision</a></li>
+  <li><a href="#roadmap" id="markdown-toc-roadmap">Roadmap</a></li>
+</ul>
+
+<h2 id="vision">Vision</h2>
+
+<p>The Machine Learning (ML) library for Flink is a new effort to bring scalable ML tools to the Flink
+community. Our goal is is to design and implement a system that is scalable and can deal with
+problems of various sizes, whether your data size is measured in megabytes or terabytes and beyond.
+We call this library FlinkML.</p>
+
+<p>An important concern for developers of ML systems is the amount of glue code that developers are
+forced to write [1] in the process of implementing an end-to-end ML system. Our goal with FlinkML
+is to help developers keep glue code to a minimum. The Flink ecosystem provides a great setting to
+tackle this problem, with its scalable ETL capabilities that can be easily combined inside the same
+program with FlinkML, allowing the development of robust pipelines without the need to use yet
+another technology for data ingestion and data munging.</p>
+
+<p>Another goal for FlinkML is to make the library easy to use. To that end we will be providing
+detailed documentation along with examples for every part of the system. Our aim is that developers
+will be able to get started with writing their ML pipelines quickly, using familiar programming
+concepts and terminology.</p>
+
+<p>Contrary to other data-processing systems, Flink exploits in-memory data streaming, and natively
+executes iterative processing algorithms which are common in ML. We plan to exploit the streaming
+nature of Flink, and provide functionality designed specifically for data streams.</p>
+
+<p>FlinkML will allow data scientists to test their models locally and using subsets of data, and then
+use the same code to run their algorithms at a much larger scale in a cluster setting.</p>
+
+<p>We are inspired by other open source efforts to provide ML systems, in particular
+<a href="http://scikit-learn.org/">scikit-learn</a> for cleanly specifying ML pipelines, and Spark’s
+<a href="https://spark.apache.org/mllib/">MLLib</a> for providing ML algorithms that scale with problem and
+cluster sizes.</p>
+
+<h2 id="roadmap">Roadmap</h2>
+
+<p>The roadmap below can provide an indication of the algorithms we aim to implement in the coming
+months. If you are interested in helping out, please check our <a href="contribution_guide.html">contribution guide</a>.
+Items in <strong>bold</strong> have already been implemented:</p>
+
+<ul>
+  <li>Pipelines of transformers and learners</li>
+  <li>Data pre-processing
+    <ul>
+      <li><strong>Feature scaling</strong></li>
+      <li><strong>Polynomial feature base mapper</strong></li>
+      <li>Feature hashing</li>
+      <li>Feature extraction for text</li>
+      <li>Dimensionality reduction</li>
+    </ul>
+  </li>
+  <li>Model selection and performance evaluation
+    <ul>
+      <li>Cross-validation for model selection and evaluation</li>
+    </ul>
+  </li>
+  <li>Supervised learning
+    <ul>
+      <li>Optimization framework
+        <ul>
+          <li><strong>Stochastic Gradient Descent</strong></li>
+          <li>L-BFGS</li>
+        </ul>
+      </li>
+      <li>Generalized Linear Models
+        <ul>
+          <li><strong>Multiple linear regression</strong></li>
+          <li>LASSO, Ridge regression</li>
+          <li>Multi-class Logistic regression</li>
+        </ul>
+      </li>
+      <li>Random forests</li>
+      <li><strong>Support Vector Machines</strong></li>
+    </ul>
+  </li>
+  <li>Unsupervised learning
+    <ul>
+      <li>Clustering
+        <ul>
+          <li>K-means clustering</li>
+        </ul>
+      </li>
+      <li>PCA</li>
+    </ul>
+  </li>
+  <li>Recommendation
+    <ul>
+      <li><strong>ALS</strong></li>
+    </ul>
+  </li>
+  <li>Text analytics
+    <ul>
+      <li>LDA</li>
+    </ul>
+  </li>
+  <li>Statistical estimation tools</li>
+  <li>Distributed linear algebra</li>
+  <li>Streaming ML</li>
+</ul>
+
+<p><strong>References:</strong></p>
+
+<p>[1] D. Sculley, G. Holt, D. Golovin, E. Davydov, T. Phillips, D. Ebner, V. Chaudhary,
+and M. Young. <em>Machine learning: The high interest credit card of technical debt.</em> In SE4ML:
+Software Engineering for Machine Learning (NIPS 2014 Workshop), 2014.</p>
+
+  </div>
+
+  <div class="col-sm-10 col-sm-offset-1">
+    <!-- Disqus thread and some vertical offset -->
+    <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div>
+  </div>
+</div>
+
+    </div><!-- /.container -->
+
+    <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
+    <!-- Include all compiled plugins (below), or include individual files as needed -->
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
+    <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js"></script>
+
+    <!-- Google Analytics -->
+    <script>
+      (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+      })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+      ga('create', 'UA-52545728-1', 'auto');
+      ga('send', 'pageview');
+    </script>
+
+    <!-- Disqus -->
+    <script type="text/javascript">
+    var disqus_shortname = 'stratosphere-eu';
+    (function() {
+        var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
+        dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
+        (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
+    })();
+</script>
+  </body>
+</html>


Mime
View raw message