commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pste...@apache.org
Subject svn commit: r1040471 - in /commons/proper/math: branches/MATH_2_X/src/main/java/org/apache/commons/math/distribution/ branches/MATH_2_X/src/site/xdoc/ branches/MATH_2_X/src/test/java/org/apache/commons/math/distribution/ trunk/src/main/java/org/apache/...
Date Tue, 30 Nov 2010 11:55:22 GMT
Author: psteitz
Date: Tue Nov 30 11:55:22 2010
New Revision: 1040471

URL: http://svn.apache.org/viewvc?rev=1040471&view=rev
Log:
Modified NormalDistributionImpl.cumulativeProbability to return 0 or 1,
respectively for values more than 40 standard deviations from the mean.
For these values, the actual probability is indistinguishable from 0 or 1
as a double.  Top coding improves performance for extreme values and prevents
convergence exceptions.

JIRA: MATH-414

Modified:
    commons/proper/math/branches/MATH_2_X/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java
    commons/proper/math/branches/MATH_2_X/src/site/xdoc/changes.xml
    commons/proper/math/branches/MATH_2_X/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java
    commons/proper/math/trunk/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java
    commons/proper/math/trunk/src/site/xdoc/changes.xml
    commons/proper/math/trunk/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java

Modified: commons/proper/math/branches/MATH_2_X/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java
URL: http://svn.apache.org/viewvc/commons/proper/math/branches/MATH_2_X/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java?rev=1040471&r1=1040470&r2=1040471&view=diff
==============================================================================
--- commons/proper/math/branches/MATH_2_X/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java
(original)
+++ commons/proper/math/branches/MATH_2_X/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java
Tue Nov 30 11:55:22 2010
@@ -171,25 +171,20 @@ public class NormalDistributionImpl exte
 
     /**
      * For this distribution, X, this method returns P(X &lt; <code>x</code>).
+     * If <code>x</code>is more than 40 standard deviations from the mean, 0
or 1 is returned,
+     * as in these cases the actual value is within <code>Double.MIN_VALUE</code>
of 0 or 1.
+     * 
      * @param x the value at which the CDF is evaluated.
      * @return CDF evaluated at <code>x</code>.
-     * @throws MathException if the algorithm fails to converge; unless
-     * x is more than 20 standard deviations from the mean, in which case the
-     * convergence exception is caught and 0 or 1 is returned.
+     * @throws MathException if the algorithm fails to converge
      */
     public double cumulativeProbability(double x) throws MathException {
-        try {
-            return 0.5 * (1.0 + Erf.erf((x - mean) /
-                    (standardDeviation * FastMath.sqrt(2.0))));
-        } catch (MaxIterationsExceededException ex) {
-            if (x < (mean - 20 * standardDeviation)) { // JDK 1.5 blows at 38
-                return 0.0d;
-            } else if (x > (mean + 20 * standardDeviation)) {
-                return 1.0d;
-            } else {
-                throw ex;
-            }
+        final double dev = x - mean;
+        if (FastMath.abs(dev) > 40 * standardDeviation) { 
+            return dev < 0 ? 0.0d : 1.0d;
         }
+        return 0.5 * (1.0 + Erf.erf((dev) /
+                    (standardDeviation * FastMath.sqrt(2.0))));
     }
 
     /**

Modified: commons/proper/math/branches/MATH_2_X/src/site/xdoc/changes.xml
URL: http://svn.apache.org/viewvc/commons/proper/math/branches/MATH_2_X/src/site/xdoc/changes.xml?rev=1040471&r1=1040470&r2=1040471&view=diff
==============================================================================
--- commons/proper/math/branches/MATH_2_X/src/site/xdoc/changes.xml (original)
+++ commons/proper/math/branches/MATH_2_X/src/site/xdoc/changes.xml Tue Nov 30 11:55:22 2010
@@ -52,6 +52,13 @@ The <action> type attribute can be add,u
     If the output is not quite correct, check for invisible trailing spaces!
      -->
     <release version="2.2" date="TBD" description="TBD">
+      <action dev="psteitz" type="fix" issue="MATH-414">
+        Modified NormalDistributionImpl.cumulativeProbability to return 0 or 1,
+        respectively for values more than 40 standard deviations from the mean.
+        For these values, the actual probability is indistinguishable from 0 or 1
+        as a double.  Top coding improves performance for extreme values and prevents
+        convergence exceptions.
+      </action>
       <action dev="psteitz" type="update" issue="MATH-420">
         Added toString() override to StatisticalSummaryValues.
       </action>

Modified: commons/proper/math/branches/MATH_2_X/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java
URL: http://svn.apache.org/viewvc/commons/proper/math/branches/MATH_2_X/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java?rev=1040471&r1=1040470&r2=1040471&view=diff
==============================================================================
--- commons/proper/math/branches/MATH_2_X/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java
(original)
+++ commons/proper/math/branches/MATH_2_X/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java
Tue Nov 30 11:55:22 2010
@@ -164,16 +164,18 @@ public class NormalDistributionTest exte
 
     /**
      * Check to make sure top-coding of extreme values works correctly.
-     * Verifies fix for JIRA MATH-167
+     * Verifies fixes for JIRA MATH-167, MATH-414
      */
     public void testExtremeValues() throws Exception {
         NormalDistribution distribution = (NormalDistribution) getDistribution();
         distribution.setMean(0);
         distribution.setStandardDeviation(1);
-        for (int i = 0; i < 100; i+=5) { // make sure no convergence exception
+        for (int i = 0; i < 100; i++) { // make sure no convergence exception
             double lowerTail = distribution.cumulativeProbability(-i);
             double upperTail = distribution.cumulativeProbability(i);
-            if (i < 10) { // make sure not top-coded
+            if (i < 9) { // make sure not top-coded 
+                // For i = 10, due to bad tail precision in erf (MATH-364), 1 is returned
+                // TODO: once MATH-364 is resolved, replace 9 with 30
                 assertTrue(lowerTail > 0.0d);
                 assertTrue(upperTail < 1.0d);
             }
@@ -182,6 +184,12 @@ public class NormalDistributionTest exte
                 assertTrue(upperTail > 0.99999);
             }
         }
+        
+        assertEquals(distribution.cumulativeProbability(Double.MAX_VALUE), 1, 0);
+        assertEquals(distribution.cumulativeProbability(-Double.MAX_VALUE), 0, 0);
+        assertEquals(distribution.cumulativeProbability(Double.POSITIVE_INFINITY), 1, 0);
+        assertEquals(distribution.cumulativeProbability(Double.NEGATIVE_INFINITY), 0, 0);
+        
    }
 
     public void testMath280() throws MathException {

Modified: commons/proper/math/trunk/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java
URL: http://svn.apache.org/viewvc/commons/proper/math/trunk/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java?rev=1040471&r1=1040470&r2=1040471&view=diff
==============================================================================
--- commons/proper/math/trunk/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java
(original)
+++ commons/proper/math/trunk/src/main/java/org/apache/commons/math/distribution/NormalDistributionImpl.java
Tue Nov 30 11:55:22 2010
@@ -114,26 +114,20 @@ public class NormalDistributionImpl exte
 
     /**
      * For this distribution, {@code X}, this method returns {@code P(X < x)}.
+     * If {@code x}is more than 40 standard deviations from the mean, 0 or 1 is returned,
+     * as in these cases the actual value is within {@code Double.MIN_VALUE} of 0 or 1.
      *
      * @param x Value at which the CDF is evaluated.
      * @return CDF evaluated at {@code x}.
-     * @throws MathException if the algorithm fails to converge; unless
-     * {@code x} is more than 20 standard deviations from the mean, in which
-     * case the convergence exception is caught and 0 or 1 is returned.
+     * @throws MathException if the algorithm fails to converge
      */
     public double cumulativeProbability(double x) throws MathException {
-        try {
-            return 0.5 * (1.0 + Erf.erf((x - mean) /
-                    (standardDeviation * FastMath.sqrt(2.0))));
-        } catch (MaxIterationsExceededException ex) {
-            if (x < (mean - 20 * standardDeviation)) { // JDK 1.5 blows at 38
-                return 0;
-            } else if (x > (mean + 20 * standardDeviation)) {
-                return 1;
-            } else {
-                throw ex;
-            }
+        final double dev = x - mean;
+        if (FastMath.abs(dev) > 40 * standardDeviation) { 
+            return dev < 0 ? 0.0d : 1.0d;
         }
+        return 0.5 * (1.0 + Erf.erf((dev) /
+                    (standardDeviation * FastMath.sqrt(2.0))));
     }
 
     /**

Modified: commons/proper/math/trunk/src/site/xdoc/changes.xml
URL: http://svn.apache.org/viewvc/commons/proper/math/trunk/src/site/xdoc/changes.xml?rev=1040471&r1=1040470&r2=1040471&view=diff
==============================================================================
--- commons/proper/math/trunk/src/site/xdoc/changes.xml (original)
+++ commons/proper/math/trunk/src/site/xdoc/changes.xml Tue Nov 30 11:55:22 2010
@@ -118,6 +118,13 @@ The <action> type attribute can be add,u
       </action>
     </release>
     <release version="2.2" date="TBD" description="TBD">
+      <action dev="psteitz" type="fix" issue="MATH-414">
+        Modified NormalDistributionImpl.cumulativeProbability to return 0 or 1,
+        respectively for values more than 40 standard deviations from the mean.
+        For these values, the actual probability is indistinguishable from 0 or 1
+        as a double.  Top coding improves performance for extreme values and prevents
+        convergence exceptions.
+      </action>
       <action dev="psteitz" type="update" issue="MATH-420">
         Added toString() override to StatisticalSummaryValues.
       </action>

Modified: commons/proper/math/trunk/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java
URL: http://svn.apache.org/viewvc/commons/proper/math/trunk/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java?rev=1040471&r1=1040470&r2=1040471&view=diff
==============================================================================
--- commons/proper/math/trunk/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java
(original)
+++ commons/proper/math/trunk/src/test/java/org/apache/commons/math/distribution/NormalDistributionTest.java
Tue Nov 30 11:55:22 2010
@@ -152,14 +152,16 @@ public class NormalDistributionTest exte
 
     /**
      * Check to make sure top-coding of extreme values works correctly.
-     * Verifies fix for JIRA MATH-167
+     * Verifies fixes for JIRA MATH-167, MATH-414
      */
     public void testExtremeValues() throws Exception {
         NormalDistribution distribution = new NormalDistributionImpl(0, 1);
-        for (int i = 0; i < 100; i+=5) { // make sure no convergence exception
+        for (int i = 0; i < 100; i++) { // make sure no convergence exception
             double lowerTail = distribution.cumulativeProbability(-i);
             double upperTail = distribution.cumulativeProbability(i);
-            if (i < 10) { // make sure not top-coded
+            if (i < 9) { // make sure not top-coded 
+                // For i = 10, due to bad tail precision in erf (MATH-364), 1 is returned
+                // TODO: once MATH-364 is resolved, replace 9 with 30
                 assertTrue(lowerTail > 0.0d);
                 assertTrue(upperTail < 1.0d);
             }
@@ -168,6 +170,12 @@ public class NormalDistributionTest exte
                 assertTrue(upperTail > 0.99999);
             }
         }
+        
+        assertEquals(distribution.cumulativeProbability(Double.MAX_VALUE), 1, 0);
+        assertEquals(distribution.cumulativeProbability(-Double.MAX_VALUE), 0, 0);
+        assertEquals(distribution.cumulativeProbability(Double.POSITIVE_INFINITY), 1, 0);
+        assertEquals(distribution.cumulativeProbability(Double.NEGATIVE_INFINITY), 0, 0);
+        
    }
 
     public void testMath280() throws MathException {



Mime
View raw message