commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
Subject [math] added a least squares section in the user guide
Date Mon, 27 Jul 2015 20:14:39 GMT
Repository: commons-math
Updated Branches:
  refs/heads/master 09fe956a6 -> 2309f28e3

added a least squares section in the user guide


Branch: refs/heads/master
Commit: 2309f28e3d6b81611d793bb5b583a11a48ef4a20
Parents: 09fe956
Author: Luc Maisonobe <>
Authored: Mon Jul 27 22:14:17 2015 +0200
Committer: Luc Maisonobe <>
Committed: Mon Jul 27 22:14:17 2015 +0200

 src/site/site.xml                        |   6 +-
 src/site/xdoc/userguide/exceptions.xml   |  10 +-
 src/site/xdoc/userguide/filter.xml       |   6 +-
 src/site/xdoc/userguide/fitting.xml      |   8 +-
 src/site/xdoc/userguide/genetics.xml     |   8 +-
 src/site/xdoc/userguide/index.xml        |  64 +++--
 src/site/xdoc/userguide/leastsquares.xml | 366 ++++++++++++++++++++++++++
 src/site/xdoc/userguide/ode.xml          |  12 +-
 src/site/xdoc/userguide/optimization.xml |   7 +-
 9 files changed, 436 insertions(+), 51 deletions(-)
diff --git a/src/site/site.xml b/src/site/site.xml
index 6f45d47..b6e0f7a 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -67,13 +67,15 @@
       <item name="Distributions"           href="/userguide/distribution.html"/>
       <item name="Fractions"               href="/userguide/fraction.html"/>
       <item name="Transform Methods"       href="/userguide/transform.html"/>
-      <item name="3D Geometry"             href="/userguide/geometry.html"/>
+      <item name="Geometry"                href="/userguide/geometry.html"/>
       <item name="Optimization"            href="/userguide/optimization.html"/>
+      <item name="Curve Fitting"           href="/userguide/fitting.html"/>
+      <item name="Least Squares"           href="/userguide/leastsquares.html"/>
       <item name="Ordinary Differential Equations" href="/userguide/ode.html"/>
       <item name="Genetic Algorithms"      href="/userguide/genetics.html"/>
       <item name="Filters"                 href="/userguide/filter.html"/>
-      <item name="Fitting"                 href="/userguide/fitting.html"/>
       <item name="Machine Learning"        href="/userguide/ml.html"/>
+      <item name="Exceptions"              href="/userguide/exceptions.html"/>
diff --git a/src/site/xdoc/userguide/exceptions.xml b/src/site/xdoc/userguide/exceptions.xml
index d16781a..4376116 100644
--- a/src/site/xdoc/userguide/exceptions.xml
+++ b/src/site/xdoc/userguide/exceptions.xml
@@ -23,14 +23,14 @@
     <title>The Commons Math User Guide - Exceptions</title>
-    <section name="16 Exceptions">
+    <section name="19 Exceptions">
-      <subsection name="16.1 Overview" href="overview">
+      <subsection name="19.1 Overview" href="overview">
         Commons Math defines a set of exceptions in order to convey the
         precise low-level cause of failure.
-      <subsection name="16.2 Unchecked Exceptions" href="unchecked">
+      <subsection name="19.2 Unchecked Exceptions" href="unchecked">
           Starting from version 3.0, all exceptions generated by the
           Commons Math code are <em>unchecked</em> (i.e. they inherit from
@@ -46,7 +46,7 @@
-      <subsection name="16.3 Hierarchies" href="hierarchies">
+      <subsection name="19.3 Hierarchies" href="hierarchies">
           The exceptions defined by Commons Math follow the Java standard
@@ -88,7 +88,7 @@
-      <subsection name="16.4 Features" href="features">
+      <subsection name="19.4 Features" href="features">
diff --git a/src/site/xdoc/userguide/filter.xml b/src/site/xdoc/userguide/filter.xml
index a241bea..b298953 100644
--- a/src/site/xdoc/userguide/filter.xml
+++ b/src/site/xdoc/userguide/filter.xml
@@ -23,13 +23,13 @@
     <title>The Commons Math User Guide - Filters</title>
-    <section name="15 Filters">
-      <subsection name="15.1 Overview" href="overview">
+    <section name="17 Filters">
+      <subsection name="17.1 Overview" href="overview">
           The filter package currently provides only an implementation of a Kalman filter.
-      <subsection name="15.2 Kalman Filter" href="kalman">
+      <subsection name="17.2 Kalman Filter" href="kalman">
           <a href="../apidocs/org/apache/commons/math4/filter/KalmanFilter.html">
           KalmanFilter</a> provides a discrete-time filter to estimate
diff --git a/src/site/xdoc/userguide/fitting.xml b/src/site/xdoc/userguide/fitting.xml
index c7c8bb1..6caace8 100644
--- a/src/site/xdoc/userguide/fitting.xml
+++ b/src/site/xdoc/userguide/fitting.xml
@@ -25,8 +25,8 @@
-    <section name="17 Curve Fitting">
-      <subsection name="17.1 Overview" href="overview">
+    <section name="13 Curve Fitting">
+      <subsection name="13.1 Overview" href="overview">
           The fitting package deals with curve fitting for univariate real functions.
           When a univariate real function y = f(x) does depend on some unknown parameters
@@ -76,7 +76,7 @@
-      <subsection name="17.2 Implemented Functions" href="special">
+      <subsection name="13.2 Implemented Functions" href="special">
           Fitting of specific functions are provided through the following classes:
@@ -128,7 +128,7 @@ final double[] coeff =;
-      <subsection name="17.3 General Case" href="general">
+      <subsection name="13.3 General Case" href="general">
           The <a href="../apidocs/org/apache/commons/math4/fitting/AbstractCurveFitter.html">
             AbstractCurveFitter</a> class provides the basic functionality for implementing
diff --git a/src/site/xdoc/userguide/genetics.xml b/src/site/xdoc/userguide/genetics.xml
index 49fb749..1455a8d 100644
--- a/src/site/xdoc/userguide/genetics.xml
+++ b/src/site/xdoc/userguide/genetics.xml
@@ -23,14 +23,14 @@
     <title>The Commons Math User Guide - Genetic Algorithms</title>
-    <section name="14 Genetic Algorithms">
-      <subsection name="14.1 Overview" href="overview">
+    <section name="16 Genetic Algorithms">
+      <subsection name="16.1 Overview" href="overview">
           The genetics package provides a framework and implementations for 
           genetic algorithms.
-      <subsection name="14.2 GA Framework">
+      <subsection name="16.2 GA Framework">
       <a href="../apidocs/org/apache/commons/math4/genetics/GeneticAlgorithm.html">
       GeneticAlgorithm</a> provides an execution framework for Genetic Algorithms (GA).
@@ -76,7 +76,7 @@
-      <subsection name="14.3 Implementation">
+      <subsection name="16.3 Implementation">
       Here is an example GA execution:
diff --git a/src/site/xdoc/userguide/index.xml b/src/site/xdoc/userguide/index.xml
index a80f40a..0dfd8e6 100644
--- a/src/site/xdoc/userguide/index.xml
+++ b/src/site/xdoc/userguide/index.xml
@@ -128,39 +128,43 @@
                 <li><a href="optimization.html#a12.4_Direct_Methods">12.4 Direct
                 <li><a href="optimization.html#a12.5_General_Case">12.5 General
-        <li><a href="ode.html">13. Ordinary Differential Equations Integration</a>
-                <ul>
-                <li><a href="ode.html#a13.1_Overview">13.1 Overview</a></li>
-                <li><a href="ode.html#a13.2_Continuous_Output">13.2 Continuous
-                <li><a href="ode.html#a13.3_Discrete_Events_Handling">13.3 Discrete
Events Handling</a></li>
-                <li><a href="ode.html#a13.4_Available_Integrators">13.4 Available
-                <li><a href="ode.html#a13.5_Derivatives">13.5 Derivatives</a></li>
-                </ul></li>
-        <li><a href="genetics.html">14. Genetic Algorithms</a>   
-                <ul>
-                <li><a href="genetics.html#a14.1_Overview">14.1 Overview</a></li>
-                <li><a href="genetics.html#a14.2_GA_Framework">14.2 GA Framework</a></li>
-                <li><a href="genetics.html#a14.3_Implementation">14.3 Implementation
and Examples</a></li>  
-        </ul></li>
-        <li><a href="filter.html">15. Filters</a>
+        <li><a href="fitting.html">13. Curve Fitting</a>
-            <li><a href="filter.html#a15.1_Overview">15.1 Overview</a></li>
-            <li><a href="filter.html#a15.2_Kalman_Filter">15.2 Kalman Filter</a></li>
+            <li><a href="fitting.html#a13.1_Overview">13.1 Overview</a></li>
+            <li><a href="fitting.html#a13.2_Implemented_Functions">13.2 Implemented
+            <li><a href="fitting.html#a13.3_General_Case">13.3 General Case</a></li>
-        <li><a href="exceptions.html">16. Exceptions</a>
+        <li><a href="leastsquares.html">14. Least Squares</a>
-            <li><a href="exceptions.html#a16.1_Overview">16.1 Overview</a></li>
-            <li><a href="exceptions.html#a16.2_Unchecked_Exceptions">16.2 Unchecked
-            <li><a href="exceptions.html#a16.3_Hierarchies">16.3 Hierarchies</a></li>
-            <li><a href="exceptions.html#a16.4_Features">16.4 Features</a></li>
+            <li><a href="fitting.html#a14.1_Overview">14.1 Overview</a></li>
+            <li><a href="fitting.html#a14.2_LeastSquaresBuilder_and_LeastSquaresFactory">14.2
LeastSquaresBuilder and LeastSquaresFactory</a></li>
+            <li><a href="fitting.html#a14.3_Model_Function">14.3 Model Function</a></li>
+            <li><a href="fitting.html#a14.4_Parameters_Validation">14.4 Parameters
+            <li><a href="fitting.html#a14.5_Tuning">14.5 Tuning</a></li>
+            <li><a href="fitting.html#a14.6_Optimization_Engine">14.6 Optimization
+            <li><a href="fitting.html#a14.7_Solving">14.7 Solving</a></li>
+            <li><a href="fitting.html#a14.8_Example">14.8 Example</a></li>
-        <li><a href="fitting.html">17. Curve Fitting</a>
+        <li><a href="ode.html">15. Ordinary Differential Equations Integration</a>
+                <ul>
+                <li><a href="ode.html#a15.1_Overview">15.1 Overview</a></li>
+                <li><a href="ode.html#a15.2_Continuous_Output">15.2 Continuous
+                <li><a href="ode.html#a15.3_Discrete_Events_Handling">15.3 Discrete
Events Handling</a></li>
+                <li><a href="ode.html#a15.4_Available_Integrators">15.4 Available
+                <li><a href="ode.html#a15.5_Derivatives">15.5 Derivatives</a></li>
+                </ul></li>
+        <li><a href="genetics.html">16. Genetic Algorithms</a>   
+                <ul>
+                <li><a href="genetics.html#a16.1_Overview">16.1 Overview</a></li>
+                <li><a href="genetics.html#a16.2_GA_Framework">16.2 GA Framework</a></li>
+                <li><a href="genetics.html#a16.3_Implementation">16.3 Implementation
and Examples</a></li>  
+        </ul></li>
+        <li><a href="filter.html">17. Filters</a>
-            <li><a href="fitting.html#a17.1_Overview">17.1 Overview</a></li>
-            <li><a href="fitting.html#a17.2_General_Case">17.2 General Case</a></li>
-            <li><a href="fitting.html#a17.3_Special_Cases">17.3 Special Cases</a></li>
+            <li><a href="filter.html#a17.1_Overview">17.1 Overview</a></li>
+            <li><a href="filter.html#a17.2_Kalman_Filter">17.2 Kalman Filter</a></li>
         <li><a href="ml.html">18. Machine Learning</a>
@@ -170,6 +174,14 @@
             <li><a href="ml.html#implementation">18.3 Implementation</a></li>
+        <li><a href="exceptions.html">19. Exceptions</a>
+          <ul>
+            <li><a href="exceptions.html#a19.1_Overview">19.1 Overview</a></li>
+            <li><a href="exceptions.html#a19.2_Unchecked_Exceptions">19.2 Unchecked
+            <li><a href="exceptions.html#a19.3_Hierarchies">19.3 Hierarchies</a></li>
+            <li><a href="exceptions.html#a19.4_Features">19.4 Features</a></li>
+          </ul>
+        </li>
diff --git a/src/site/xdoc/userguide/leastsquares.xml b/src/site/xdoc/userguide/leastsquares.xml
new file mode 100644
index 0000000..8da9e78
--- /dev/null
+++ b/src/site/xdoc/userguide/leastsquares.xml
@@ -0,0 +1,366 @@
+<?xml version="1.0"?>
+   Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+  -->
+<?xml-stylesheet type="text/xsl" href="./xdoc.xsl"?>
+<document url="fitting.html">
+  <properties>
+    <title>The Commons Math User Guide - Least squares</title>
+  </properties>
+  <body>
+    <section name="14 Least squares">
+      <subsection name="14.1 Overview">
+        <p>
+          The least squares package fits a parametric model to a set of observed
+          values by minimizing a cost function with a specific form.
+          The fitting basically consists in finding the values
+          for some parameters p<sub>k</sub> such that a cost function
+          J = sum(w<sub>i</sub>(target<sub>i</sub> - model<sub>i</sub>)<sup>2</sup>)
+          minimized. The various (target<sub>i</sub> - model<sub>i</sub>(p<sub>k</sub>))
+          terms are called residuals. They represent the deviation between a set of
+          target values target<sub>i</sub> and theoretical values computed from
+          models model<sub>i</sub> depending on free parameters p<sub>k</sub>.
+          The w<sub>i</sub> factors are weights. One classical use case is when
+          target values are experimental observations or measurements.
+        </p>
+        <p>
+          Two engines devoted to least-squares problems are available. The first one is
+          based on the <a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/GaussNewtonOptimizer.html">
+          Gauss-Newton</a> method. The second one is the <a
+          href="../apidocs/org/apache/commons/math4/fitting/leastsquares/LevenbergMarquardtOptimizer.html">
+          Levenberg-Marquardt</a> method.
+        </p>
+      </subsection>
+      <subsection name="14.2 LeastSquaresBuilder and LeastSquaresFactory">
+        <p>
+          In order to solve a least-squares fitting problem, the user must provide the following
+          <ul>
+           <li>a mean to evaluate all the components of the model for a given set of
+               model<sub>i</sub> = f<sub>i</sub>(p<sub>1</sub>,
p<sub>2</sub>, ... p<sub>k</sub>),
+               this is code</li>
+           <li>the target (or observed) components: target<sub>i</sub>,
this is data</li>
+           <li>the start values for all p<sub>k</sub> parameters: s<sub>k</sub>,
this is data</li>
+           <li>optionally a validator for the p<sub>k</sub> parameters,
this is code</li>
+           <li>optionally the weight for sample point i: w<sub>i</sub>,
this is data and defaults to 1.0 if not provided</li>
+           <li>a maximum number of iterations, this is data</li>
+           <li>a maximum number of evaluations, this is data</li>
+           <li>a convergence criterion, this is code</li>
+          </ul>
+        </p>
+        <p>
+          The elements of the list above can be provided as an implementation of the
+          <a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/LeastSquaresProblem.html">
+          LeastSquaresProblem</a> interface. However, this is cumbersome to do directly,
so some helper
+          classes are available. The first helper is a mutable builder:
+          <a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/LeastSquaresBuilder.html">
+          LeastSquaresBuilder</a>. The second helper is an utility factory:
+          <a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/LeastSquaresFactory.html">
+          LeastSquaresFactory</a>.
+        </p>
+        <p>
+          The builder class is better suited when setting the various elements of the least
+          problem is done progressively in different places in the user code. In this case,
the user
+          would create first an empty builder andconfigure it progressively by calling its
+          (<code>start</code>, <code>target</code>, <code>model</code>,
...). Once the configuration
+          is complete, calling the <code>build</code> method would create the
least squares problem.
+        </p>
+        <p>
+          The factory utility is better suited when the various elements of the least squares
+          problem are all known at one place and the problem can be built in just one sweep,
+          to one of the static <code>LeastSquaresFactory.create</code> method.
+        </p>
+      </subsection>
+      <subsection name="14.3 Model Function">
+        <p>
+          The model function is used by the least squares engine to evaluate the model components
+          model<sub>i</sub> given some test parameters p<sub>k</sub>.
It is therefore a multivariate
+          function (it depends on the various p<sub>k</sub>) and it is vector-valued
(it has several
+          components model<sub>i</sub>). There must be exactly one component
model<sub>i</sub> for
+          each target (or observed) component target<sub>i</sub>, otherwise some
residuals to be
+          squared and summed could not be computed. In order for the problem to be well defined,
+          number of parameters p<sub>k</sub> must be less than the number of
components model<sub>i</sub>.
+          Failing to ensure this may lead to the engine throwing an exception as the underlying
+          algebra operations may encounter singular matrices. It is not unusual to have a
large number
+          of components (several thousands) and only a dozen parameters. There are no limitations
on these
+          numbers, though.
+        </p>
+        <p>
+          As the least squares engine needs to create Jacobians matrices for the model function,
+          its value and its derivatives <em>with respect to the p<sub>k</sub>
parameters</em> must
+          be available. There are two ways to provide this:
+          <ul>
+            <li>provide one
+            <a href="../apidocs/org/apache/commons/math4/analysis/MultivariateVectorFunction.html">MultivariateVectorFunction</a>
+            instance for computing the components values and one
+            <a href="../apidocs/org/apache/commons/math4/analysis/MultivariateMatrixFunction.html">MultivariateMatrixFunction</a>
+            instance for computing the components derivatives (i.e. the Jacobian matrix)
+            respect to the parameters,</li>
+            <li>or provide one
+            <a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/MultivariateJacobianFunction.html">MultivariateJacobianFunction</a>
+            instance for computing both the components values and their derivatives simultaneously.</li>
+          </ul>
+          The first alternative is best suited for models which are not computationally intensive
+          as it allows more modularized code with one method for each type of computation.
The second
+          alternative is best suited for models which are computationally intensive and evaluating
+          both the values and derivatives in one sweep saves a lot of work.
+        </p>
+        <p>
+          The <code>point</code> parameter of the <code>value</code>
methods in the
+          <a href="../apidocs/org/apache/commons/math4/analysis/MultivariateVectorFunction.html">MultivariateVectorFunction</a>,
+          <a href="../apidocs/org/apache/commons/math4/analysis/MultivariateMatrixFunction.html">MultivariateMatrixFunction</a>,
+          or <a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/MultivariateJacobianFunction.html">MultivariateJacobianFunction</a>
+          interfaces will contain the parameters p<sub>k</sub>. The values will
be the model components
+          model<sub>i</sub> and the derivatives will be the derivatives of the
model components
+          with respect to the parameters dmodel<sub>i</sub>/dp<sub>k</sub>.
+        </p>
+        <p>
+          There are no requirements on how to compute value and derivatives. The
+          <a href="../apidocs/org/apache/commons/math4/analysis/differentiation/DerivativeStructure.html">
+          DerivativeStructure</a> class may be useful to compute analytically derivatives
+          difficult cases, but this class is not mandated by the API which only expects the
+          as a Jacobian matrix containing primitive double entries.
+        </p>
+        <p>
+          One non-obvious feature provided by both the builder and the factory is lazy evaluation.
This feature
+          allows to defer calls to the model functions until they are really needed by the
engine. This
+          can save some calls for engines that evaluate the value and the Jacobians in different
+          (this is the case for Levenberg-Marquardt). However, lazy evaluation is possible
+          if the model functions are themselves separated, i.e. it can be used only with
the first
+          alternative above. Setting up the <code>lazyEvaluation</code> flag
to <code>true</code> in the builder
+          or factory and setting up the model function as one
+          <a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/MultivariateJacobianFunction.html">MultivariateJacobianFunction</a>
+          instance at the same time will trigger an illegal state exception telling that
the model function
+          misses required functionality.
+        </p>
+      </subsection>
+      <subsection name="14.4 Parameters Validation">
+       <p>
+         In some cases, the model function requires parameters to lie within a specific domain.
For example
+         a parameter may be used in a square root and needs to be positive, or another parameter
+         the sine of an angle and should be within -1 and +1, or several parameters may need
to remain in
+         the unit circle and the sum of their squares must be smaller than 1. The least square
solvers available
+         in Apache Commons Math currently don't allow to set up constraints on the parameters.
This is a
+         known missing feature. There are two ways to circumvent this.
+       </p>
+       <p>
+         Both ways are achieved by setting up a
+         <a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/ParameterValidator.html">ParameterValidator</a>
+         instance. The input of the value and jacobian model functions will always be the
output of
+         the parameter validator if one exists.
+       </p>
+       <p>
+         One way to constrain parameters is to use a continuous mapping between the parameters
that the
+         least squares solver will handle and the real parameters of the mathematical model.
Using mapping
+         functions like <code>logit</code> and <code>sigmoid</code>,
one can map a finite range to the
+         infinite real line. Using mapping functions based on <code>log</code>
and <code>exp</code>, one
+         can map a semi-infinite range to the infinite real line. It is possible to use such
a mapping so
+         that the engine will always see unbounded parameters, whereas on the other side
of the mapping the
+         mathematical model will always see parameters mapped correctly to the expected range.
Care must be
+         taken with derivatives as one must remember that the parameters have been mapped.
Care must also
+         be taken with convergence status. This may be tricky.
+       </p>
+       <p>
+         Another way to constrain parameters is to simply truncate the parameters back to
the domain when
+         one search point escapes from it and not care about derivatives. This works <em>only</em>
if the
+         solution is expected to be inside the domain and not at the boundary, as points
out of the domain
+         will only be temporary test points with a cost function higher than the real solution
and will soon
+         be dropped by the underlying engine. As a rule of thumb, these conditions are met
only when the
+         domain boundaries correspond to unrealistic values that will never be achieved (null
+         negative masses, ...) but they will not be met when the domain boundaries are more
+         limits (a maximum weight that can be handled by a device, a minimum temperature
that can be
+         sustained by an instrument, ...).
+       </p>
+      </subsection>
+      <subsection name="14.5 Tuning">
+        <p>
+          Among the elements to be provided to the least squares problem builder or factory
+          are some tuning parameters for the solver.
+        </p>
+        <p>
+          The maximum number of iterations refers to the engine algorithm main loop, whereas
+          maximum number of iterations refers to the number of calls to the model method.
+          algorithms (like Levenberg-Marquardt) have two embedded loops, with iteration number
+          being incremented at outer loop level, but a new evaluation being done at each
+          loop. In this case, the number of evaluations will be greater than the number of
+          Other algorithms (like Gauss-Newton) have only one level of loops. In this case,
+          number of evaluations will equal to the number of iterations. In any case, the
+          numbers are really only intended as safeguard to prevent infinite loops, so the
+          value of the limit is not important so it is common to select some almost arbitrary
+          much larger than the expected number of evaluations and use it for both
+          <code>maxIterations</code> and <code>maxEvaluations</code>.
As an example, if the least
+          squares solver usually finds a solution in 50 iterations, setting a maximum value
to 1000
+          is probably safe and prevents infinite loops. If the least squares solver needs
+          hundreds of evaluations, it would probably be safer to set the maximum value to
10000 or
+          even 1000000 to avoid failures in slightly more demanding cases. Very fine tuning
+          these maximum numbers is often worthless, they are only intended as safeguards.
+        </p>
+        <p>
+          Convergence checking is delegated to a dedicated interface from the <code>optim</code>
+          package: <a href="../apidocs/org/apache/commons/math4/optim/ConvergenceChecker.html">
+          ConvergenceChecker</a>, parameterized with either the specific
+          <a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/LeastSquaresProblem.Evaluation.html">Evaluation</a>
+          class used for least squares problems or the general
+          <a href="../apidocs/org/apache/commons/math4/optim/PointVectorValuePair.html">PointVectorValuePair</a>.
+          Each time convergence is checked, both the previous
+          and the current evaluations of the least squares problem are provided, so the checker
+          compare them and decide whereas convergence has been reached or not. The predefined
+          checker implementations that can be useful for least squares fitting are:
+          <ul>
+            <li><a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/EvaluationRmsChecker.html">EvaluationRmsChecker</a>,
+            which uses only the normalized cost (square-root of the sum of squared of the
+            divided by the number of measurements),</li>
+            <li><a href="../apidocs/org/apache/commons/math4/optim/SimpleVectorValueChecker.html">SimpleVectorValueChecker</a>,
+            which uses the model components themselves (<em>not</em> the residuals),</li>
+            <li><a href="../apidocs/org/apache/commons/math4/optim/SimplePointChecker.html">SimplePointChecker&lt;PointVectorValuePair&gt;</a>,
+            which uses the parameters.</li>
+          </ul>
+          Of course, users can also provide their own implementation of the
+          <a href="../apidocs/org/apache/commons/math4/optim/ConvergenceChecker.html">ConvergenceChecker</a>
+          interface.
+        </p>
+      </subsection>
+      <subsection name="14.6 Optimization Engine">
+        <p>
+          Once the least squares problem has been created, using either the builder or the
+          it is passed to an optimization engine for solving. Two engines devoted to least-squares
+          problems are available. The first one is
+          based on the <a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/GaussNewtonOptimizer.html">
+          Gauss-Newton</a> method. The second one is the <a
+          href="../apidocs/org/apache/commons/math4/fitting/leastsquares/LevenbergMarquardtOptimizer.html">
+          Levenberg-Marquardt</a> method. For both increased readability and in order
to leverage
+          possible future changes in the configuration, it is recommended to use the fluent-style
API to
+          build and configure the optimizers. This means creating a first temporary version
of the optimizer
+          with a default parameterless constructor, and then to set up the various configuration
+          using the available <code>withXxx</code> methods that all return a
new optimizer instance. Only the
+          final fully configured instance is used. As an example, setting up a Levenberg-Marquardt
+          all configuration set to default except the cost relative tolerance and parameter
relative tolerance
+          would be done as follows:
+        </p>
+        <source>
+  LeastSquaresOptimizer optimizer = new LevenbergMarquardtOptimizer().
+                                    withCostRelativeTolerance(1.0e-12).
+                                    withParameterRelativeTolerance(1.0e-12);
+        </source>
+        <p>
+        As another example, setting up a Gauss-Newton optimizer and forcing the decomposition
to SVD (the
+        default is QR decomposition) would be done as follows:
+        </p>
+        <source>
+  LeastSquaresOptimizer optimizer = new GaussNewtonOptimizer().
+                                    withwithDecomposition(GaussNewtonOptimizer.Decomposition.QR);
+        </source>
+      </subsection>
+      <subsection name="14.7 Solving">
+        <p>
+        Solving the least squares problem is done by calling the <code>optimize</code>
method of the
+        optimizer and passing the least squares problem as the single parameter:
+        </p>
+        <source>
+  LeastSquaresOptimizer.Optimum optimum = optimizer.optimize(leastSquaresProblem);
+        </source>
+        <p>
+          The <a
+          href="../apidocs/org/apache/commons/math4/fitting/leastsquares/LeastSquaresOptimizer.Optimum.html">
+          LeastSquaresOptimizer.Optimum</a> class is a specialized
+          <a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/LeastSquaresProblem.Evaluation.html">Evaluation</a>
+          with additional methods te retrieve the number of evaluations and number of iterations
+          The most important methods are inherited from the
+          <a href="../apidocs/org/apache/commons/math4/fitting/leastsquares/LeastSquaresProblem.Evaluation.html">Evaluation</a>
+          class and correspond to the point (i.e. the parameters), cost, Jacobian, RMS, covariance
+        </p>
+      </subsection>
+      <subsection name="14.8 Example">
+        <p>
+          The following simple example shows how to find the center of a circle of known
radius to
+          to best fit observed 2D points. It is a simplified version of one of the JUnit
test cases.
+          In the complete test case, both the circle center and its radius are fitted, here
+          radius is fixed.
+        </p>
+        <source>
+  final double radius = 70.0;
+  final Vector2D[] observedPoints = new Vector2D[] {
+      new Vector2D( 30.0,  68.0),
+      new Vector2D( 50.0,  -6.0),
+      new Vector2D(110.0, -20.0),
+      new Vector2D( 35.0,  15.0),
+      new Vector2D( 45.0,  97.0)
+  };
+  // the model function components are the distances to current estimated center,
+  // they should be as close as possible to the specified radius
+  MultivariateJacobianFunction distancesToCurrentCenter = new MultivariateJacobianFunction()
+      public Pair&lt;RealVector, RealMatrix&gt; value(final RealVector point) {
+          Vector2D center = new Vector2D(point.getEntry(0), point.getEntry(1));
+          RealVector value = new ArrayRealVector(observedPoints.length);
+          RealMatrix jacobian = new Array2DRowRealMatrix(observedPoints.length, 2);
+          for (int i = 0; i &lt; observedPoints.length; ++i) {
+              Vector2D o = observedPoints[i];
+              double modelI = Vector2D.distance(o, center);
+              value.setEntry(i, modelI);
+              // derivative with respect to p0 = x center
+              jacobian.setEntry(i, 0, (center.getX() - o.getX()) / modelI);
+              // derivative with respect to p1 = y center
+              jacobian.setEntry(i, 1, (center.getX() - o.getX()) / modelI);
+          }
+          return new Pair&lt;RealVector, RealMatrix&gt;(value, jacobian);
+      }
+  };
+  // the target is to have all points at the specified radius from the center
+  double[] prescribedDistances = new double[observedPoints.length];
+  Arrays.fill(prescribedDistances, radius);
+  // least squares problem to solve : modeled radius should be close to target radius
+  LeastSquaresProblem problem = new LeastSquaresBuilder().
+                                start(new double[] { 100.0, 50.0 }).
+                                model(distancesToCurrentCenter).
+                                target(prescribedDistances).
+                                lazyEvaluation(false).
+                                maxEvaluations(1000).
+                                maxIterations(1000).
+                                build();
+  LeastSquaresOptimizer.Optimum optimum = new LevenbergMarquardtOptimizer().optimize(problem);
+  Vector2D fittedCenter = new Vector2D(optimum.getPoint().getEntry(0), optimum.getPoint().getEntry(1));
+  System.out.println("fitted center: " + fittedCenter.getX() + " " + fittedCenter.getY());
+  System.out.println("RMS: "           + optimum.getRMS());
+  System.out.println("evaluations: "   + optimum.getEvaluations());
+  System.out.println("iterations: "    + optimum.getIterations());
+        </source>
+      </subsection>
+     </section>
+  </body>
diff --git a/src/site/xdoc/userguide/ode.xml b/src/site/xdoc/userguide/ode.xml
index d3b2eea..b2fe395 100644
--- a/src/site/xdoc/userguide/ode.xml
+++ b/src/site/xdoc/userguide/ode.xml
@@ -25,8 +25,8 @@
-    <section name="13 Ordinary Differential Equations Integration">
-      <subsection name="13.1 Overview" href="overview">
+    <section name="15 Ordinary Differential Equations Integration">
+      <subsection name="15.1 Overview" href="overview">
           The ode package provides classes to solve Ordinary Differential Equations problems.
@@ -115,7 +115,7 @@ double[] y = new double[] { 0.0, 1.0 }; // initial state
 dp853.integrate(ode, 0.0, y, 16.0, y); // now y contains final state at time t=16.0
-      <subsection name="13.2 Continuous Output" href="continuous">
+      <subsection name="15.2 Continuous Output" href="continuous">
           The solution of the integration problem is provided by two means. The first one
is aimed towards
           simple use: the state vector at the end of the integration process is copied in
the y array of the
@@ -178,7 +178,7 @@ integrator.addStepHandler(stepHandler);
           automatic guess is wrong.
-      <subsection name="13.3 Discrete Events Handling" href="events">
+      <subsection name="15.3 Discrete Events Handling" href="events">
           ODE problems are continuous ones. However, sometimes discrete events must be
           taken into account. The most frequent case is the stop condition of the integrator
@@ -256,7 +256,7 @@ public int eventOccurred(double t, double[] y, boolean increasing) {
-      <subsection name="13.4 Available Integrators" href="integrators">
+      <subsection name="15.4 Available Integrators" href="integrators">
           The tables below show the various integrators available for non-stiff problems.
Note that the
           implementations of Adams-Bashforth and Adams-Moulton are adaptive stepsize, not
fixed stepsize
@@ -288,7 +288,7 @@ public int eventOccurred(double t, double[] y, boolean increasing) {
-      <subsection name="13.5 Derivatives" href="derivatives">
+      <subsection name="15.5 Derivatives" href="derivatives">
           If in addition to state y(t) the user needs to compute the sensitivity of the final
state with respect to
           the initial state (dy/dy<sub>0</sub>) or the sensitivity of the final
state with respect to some parameters
diff --git a/src/site/xdoc/userguide/optimization.xml b/src/site/xdoc/userguide/optimization.xml
index f691ffa..7790db1 100644
--- a/src/site/xdoc/userguide/optimization.xml
+++ b/src/site/xdoc/userguide/optimization.xml
@@ -27,7 +27,12 @@
     <section name="12 Optimization">
       <p><em>The contents of this section currently describes deprecated classes.</em>
-        Please refer to the new <a href="../apidocs/org/apache/commons/math4/optim/package-summary.html">API
+        Please refer to the new <a href="../apidocs/org/apache/commons/math4/optim/package-summary.html">API
+        description</a>.
+      </p>
+      <p>Least squares optimizers are not in this package anymore, they have been moved
+        in a dedicated least-squares sub-package described in the <a href="./leastsquares.html">least
+        section.
       <subsection name="12.1 Overview" href="overview">

View raw message