Return-Path: 
 <commits-return-14951-apmail-commons-commits-archive=commons.apache.org@commons.apache.org>
Delivered-To: apmail-commons-commits-archive@minotaur.apache.org
Received: (qmail 35447 invoked from network); 30 Nov 2010 11:56:57 -0000
Received: from unknown (HELO mail.apache.org) (140.211.11.3)
  by 140.211.11.9 with SMTP; 30 Nov 2010 11:56:57 -0000
Received: (qmail 48373 invoked by uid 500); 30 Nov 2010 11:56:57 -0000
Delivered-To: apmail-commons-commits-archive@commons.apache.org
Received: (qmail 48050 invoked by uid 500); 30 Nov 2010 11:56:56 -0000
Mailing-List: contact commits-help@commons.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:commits-help@commons.apache.org>
List-Unsubscribe: <mailto:commits-unsubscribe@commons.apache.org>
List-Post: <mailto:commits@commons.apache.org>
List-Id: <commits.commons.apache.org>
Reply-To: dev@commons.apache.org
Delivered-To: mailing list commits@commons.apache.org
Received: (qmail 48043 invoked by uid 99); 30 Nov 2010 11:56:55 -0000
Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136)
    by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 30 Nov 2010 11:56:55 +0000
X-ASF-Spam-Status: No, hits=-2000.0 required=10.0
	tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
    by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 30 Nov 2010 11:56:54 +0000
Received: by eris.apache.org (Postfix, from userid 65534)
	id E5F982388A38; Tue, 30 Nov 2010 11:55:22 +0000 (UTC)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1040471 - in /commons/proper/math:
 branches/MATH_2_X/src/main/java/org/apache/commons/math/distribution/
 branches/MATH_2_X/src/site/xdoc/
 branches/MATH_2_X/src/test/java/org/apache/commons/math/distribution/
 trunk/src/main/java/org/apache/...
Date: Tue, 30 Nov 2010 11:55:22 -0000
To: commits@commons.apache.org
From: psteitz@apache.org
X-Mailer: svnmailer-1.0.8
Message-Id: <20101130115522.E5F982388A38@eris.apache.org>

Author: psteitz
Date: Tue Nov 30 11:55:22 2010
New Revision: 1040471

URL: http://svn.apache.org/viewvc?rev=1040471&view=rev
Log:
Modified NormalDistributionImpl.cumulativeProbability to return 0 or 1,
respectively for values more than 40 standard deviations from the mean.
For these values, the actual probability is indistinguishable from 0 or 1
as a double.  Top coding improves performance for extreme values and prevents
convergence exceptions.

JIRA: MATH-414

Modified:
    commons/proper/math/branches/MATH_2_X/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java
    commons/proper/math/branches/MATH_2_X/src/site/xdoc/changes.xml
    commons/proper/math/branches/MATH_2_X/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java
    commons/proper/math/trunk/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java
    commons/proper/math/trunk/src/site/xdoc/changes.xml
    commons/proper/math/trunk/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java

Modified: commons/proper/math/branches/MATH_2_X/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java
URL: http://svn.apache.org/viewvc/commons/proper/math/branches/MATH_2_X/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java?rev=1040471&r1=1040470&r2=1040471&view=diff
==============================================================================
--- commons/proper/math/branches/MATH_2_X/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java (original)
+++ commons/proper/math/branches/MATH_2_X/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java Tue Nov 30 11:55:22 2010
@@ -171,25 +171,20 @@ public class NormalDistributionImpl exte
 
     /**
      * For this distribution, X, this method returns P(X &lt; <code>x</code>).
+     * If <code>x</code>is more than 40 standard deviations from the mean, 0 or 1 is returned,
+     * as in these cases the actual value is within <code>Double.MIN_VALUE</code> of 0 or 1.
+     * 
      * @param x the value at which the CDF is evaluated.
      * @return CDF evaluated at <code>x</code>.
-     * @throws MathException if the algorithm fails to converge; unless
-     * x is more than 20 standard deviations from the mean, in which case the
-     * convergence exception is caught and 0 or 1 is returned.
+     * @throws MathException if the algorithm fails to converge
      */
     public double cumulativeProbability(double x) throws MathException {
-        try {
-            return 0.5 * (1.0 + Erf.erf((x - mean) /
-                    (standardDeviation * FastMath.sqrt(2.0))));
-        } catch (MaxIterationsExceededException ex) {
-            if (x < (mean - 20 * standardDeviation)) { // JDK 1.5 blows at 38
-                return 0.0d;
-            } else if (x > (mean + 20 * standardDeviation)) {
-                return 1.0d;
-            } else {
-                throw ex;
-            }
+        final double dev = x - mean;
+        if (FastMath.abs(dev) > 40 * standardDeviation) { 
+            return dev < 0 ? 0.0d : 1.0d;
         }
+        return 0.5 * (1.0 + Erf.erf((dev) /
+                    (standardDeviation * FastMath.sqrt(2.0))));
     }
 
     /**

Modified: commons/proper/math/branches/MATH_2_X/src/site/xdoc/changes.xml
URL: http://svn.apache.org/viewvc/commons/proper/math/branches/MATH_2_X/src/site/xdoc/changes.xml?rev=1040471&r1=1040470&r2=1040471&view=diff
==============================================================================
--- commons/proper/math/branches/MATH_2_X/src/site/xdoc/changes.xml (original)
+++ commons/proper/math/branches/MATH_2_X/src/site/xdoc/changes.xml Tue Nov 30 11:55:22 2010
@@ -52,6 +52,13 @@ The <action> type attribute can be add,u
     If the output is not quite correct, check for invisible trailing spaces!
      -->
     <release version="2.2" date="TBD" description="TBD">
+      <action dev="psteitz" type="fix" issue="MATH-414">
+        Modified NormalDistributionImpl.cumulativeProbability to return 0 or 1,
+        respectively for values more than 40 standard deviations from the mean.
+        For these values, the actual probability is indistinguishable from 0 or 1
+        as a double.  Top coding improves performance for extreme values and prevents
+        convergence exceptions.
+      </action>
       <action dev="psteitz" type="update" issue="MATH-420">
         Added toString() override to StatisticalSummaryValues.
       </action>

Modified: commons/proper/math/branches/MATH_2_X/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java
URL: http://svn.apache.org/viewvc/commons/proper/math/branches/MATH_2_X/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java?rev=1040471&r1=1040470&r2=1040471&view=diff
==============================================================================
--- commons/proper/math/branches/MATH_2_X/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java (original)
+++ commons/proper/math/branches/MATH_2_X/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java Tue Nov 30 11:55:22 2010
@@ -164,16 +164,18 @@ public class NormalDistributionTest exte
 
     /**
      * Check to make sure top-coding of extreme values works correctly.
-     * Verifies fix for JIRA MATH-167
+     * Verifies fixes for JIRA MATH-167, MATH-414
      */
     public void testExtremeValues() throws Exception {
         NormalDistribution distribution = (NormalDistribution) getDistribution();
         distribution.setMean(0);
         distribution.setStandardDeviation(1);
-        for (int i = 0; i < 100; i+=5) { // make sure no convergence exception
+        for (int i = 0; i < 100; i++) { // make sure no convergence exception
             double lowerTail = distribution.cumulativeProbability(-i);
             double upperTail = distribution.cumulativeProbability(i);
-            if (i < 10) { // make sure not top-coded
+            if (i < 9) { // make sure not top-coded 
+                // For i = 10, due to bad tail precision in erf (MATH-364), 1 is returned
+                // TODO: once MATH-364 is resolved, replace 9 with 30
                 assertTrue(lowerTail > 0.0d);
                 assertTrue(upperTail < 1.0d);
             }
@@ -182,6 +184,12 @@ public class NormalDistributionTest exte
                 assertTrue(upperTail > 0.99999);
             }
         }
+        
+        assertEquals(distribution.cumulativeProbability(Double.MAX_VALUE), 1, 0);
+        assertEquals(distribution.cumulativeProbability(-Double.MAX_VALUE), 0, 0);
+        assertEquals(distribution.cumulativeProbability(Double.POSITIVE_INFINITY), 1, 0);
+        assertEquals(distribution.cumulativeProbability(Double.NEGATIVE_INFINITY), 0, 0);
+        
    }
 
     public void testMath280() throws MathException {

Modified: commons/proper/math/trunk/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java
URL: http://svn.apache.org/viewvc/commons/proper/math/trunk/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java?rev=1040471&r1=1040470&r2=1040471&view=diff
==============================================================================
--- commons/proper/math/trunk/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java (original)
+++ commons/proper/math/trunk/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java Tue Nov 30 11:55:22 2010
@@ -114,26 +114,20 @@ public class NormalDistributionImpl exte
 
     /**
      * For this distribution, {@code X}, this method returns {@code P(X < x)}.
+     * If {@code x}is more than 40 standard deviations from the mean, 0 or 1 is returned,
+     * as in these cases the actual value is within {@code Double.MIN_VALUE} of 0 or 1.
      *
      * @param x Value at which the CDF is evaluated.
      * @return CDF evaluated at {@code x}.
-     * @throws MathException if the algorithm fails to converge; unless
-     * {@code x} is more than 20 standard deviations from the mean, in which
-     * case the convergence exception is caught and 0 or 1 is returned.
+     * @throws MathException if the algorithm fails to converge
      */
     public double cumulativeProbability(double x) throws MathException {
-        try {
-            return 0.5 * (1.0 + Erf.erf((x - mean) /
-                    (standardDeviation * FastMath.sqrt(2.0))));
-        } catch (MaxIterationsExceededException ex) {
-            if (x < (mean - 20 * standardDeviation)) { // JDK 1.5 blows at 38
-                return 0;
-            } else if (x > (mean + 20 * standardDeviation)) {
-                return 1;
-            } else {
-                throw ex;
-            }
+        final double dev = x - mean;
+        if (FastMath.abs(dev) > 40 * standardDeviation) { 
+            return dev < 0 ? 0.0d : 1.0d;
         }
+        return 0.5 * (1.0 + Erf.erf((dev) /
+                    (standardDeviation * FastMath.sqrt(2.0))));
     }
 
     /**

Modified: commons/proper/math/trunk/src/site/xdoc/changes.xml
URL: http://svn.apache.org/viewvc/commons/proper/math/trunk/src/site/xdoc/changes.xml?rev=1040471&r1=1040470&r2=1040471&view=diff
==============================================================================
--- commons/proper/math/trunk/src/site/xdoc/changes.xml (original)
+++ commons/proper/math/trunk/src/site/xdoc/changes.xml Tue Nov 30 11:55:22 2010
@@ -118,6 +118,13 @@ The <action> type attribute can be add,u
       </action>
     </release>
     <release version="2.2" date="TBD" description="TBD">
+      <action dev="psteitz" type="fix" issue="MATH-414">
+        Modified NormalDistributionImpl.cumulativeProbability to return 0 or 1,
+        respectively for values more than 40 standard deviations from the mean.
+        For these values, the actual probability is indistinguishable from 0 or 1
+        as a double.  Top coding improves performance for extreme values and prevents
+        convergence exceptions.
+      </action>
       <action dev="psteitz" type="update" issue="MATH-420">
         Added toString() override to StatisticalSummaryValues.
       </action>

Modified: commons/proper/math/trunk/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java
URL: http://svn.apache.org/viewvc/commons/proper/math/trunk/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java?rev=1040471&r1=1040470&r2=1040471&view=diff
==============================================================================
--- commons/proper/math/trunk/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java (original)
+++ commons/proper/math/trunk/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java Tue Nov 30 11:55:22 2010
@@ -152,14 +152,16 @@ public class NormalDistributionTest exte
 
     /**
      * Check to make sure top-coding of extreme values works correctly.
-     * Verifies fix for JIRA MATH-167
+     * Verifies fixes for JIRA MATH-167, MATH-414
      */
     public void testExtremeValues() throws Exception {
         NormalDistribution distribution = new NormalDistributionImpl(0, 1);
-        for (int i = 0; i < 100; i+=5) { // make sure no convergence exception
+        for (int i = 0; i < 100; i++) { // make sure no convergence exception
             double lowerTail = distribution.cumulativeProbability(-i);
             double upperTail = distribution.cumulativeProbability(i);
-            if (i < 10) { // make sure not top-coded
+            if (i < 9) { // make sure not top-coded 
+                // For i = 10, due to bad tail precision in erf (MATH-364), 1 is returned
+                // TODO: once MATH-364 is resolved, replace 9 with 30
                 assertTrue(lowerTail > 0.0d);
                 assertTrue(upperTail < 1.0d);
             }
@@ -168,6 +170,12 @@ public class NormalDistributionTest exte
                 assertTrue(upperTail > 0.99999);
             }
         }
+        
+        assertEquals(distribution.cumulativeProbability(Double.MAX_VALUE), 1, 0);
+        assertEquals(distribution.cumulativeProbability(-Double.MAX_VALUE), 0, 0);
+        assertEquals(distribution.cumulativeProbability(Double.POSITIVE_INFINITY), 1, 0);
+        assertEquals(distribution.cumulativeProbability(Double.NEGATIVE_INFINITY), 0, 0);
+        
    }
 
     public void testMath280() throws MathException {