db-derby-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From krist...@apache.org
Subject svn commit: r759181 - in /db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb: ClusMog.java ClusMogSQL.java GenMog.java MogTest.java SampMog.java _Suite.java
Date Fri, 27 Mar 2009 15:46:00 GMT
Author: kristwaa
Date: Fri Mar 27 15:45:59 2009
New Revision: 759181

URL: http://svn.apache.org/viewvc?rev=759181&view=rev
Log:
DERBY-4085: Improve testing of the in-memory back end.
Adds a functional test (mixture-of-Gaussian distribution).
The calculations are carried out both in Java and though SQL, then compared.
The tests are not yet compatible with JSR 169 / JavaME.

Patch contributed by Cheng Che Chen (chatom).
Minor modifications by Kristian Waagan.

Patch file: derby-4085-2a-mog_func_test.diff


Added:
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMog.java
  (with props)
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMogSQL.java
  (with props)
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/GenMog.java
  (with props)
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/MogTest.java
  (with props)
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/SampMog.java
  (with props)
Modified:
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/_Suite.java

Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMog.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMog.java?rev=759181&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMog.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMog.java
Fri Mar 27 15:45:59 2009
@@ -0,0 +1,289 @@
+/*
+   Derby - Class org.apache.derbyTesting.functionTests.tests.memorydb.ClusMog
+
+   Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License.
+ */
+
+package org.apache.derbyTesting.functionTests.tests.memorydb;
+
+import java.util.Arrays;
+import java.util.Random;
+import org.apache.derbyTesting.junit.BaseTestCase;
+
+/**
+ * Simple utility to compute/recover the parameters of a mixture-of-Gaussian
+ * distribution from independent samples.
+ */
+public class ClusMog
+{
+  /** default constructor */
+  public ClusMog() {}
+
+  /**
+   * Compute/recover the parameters of a mixture-of-Gaussian distribution
+   * from given independent samples.
+   * @param n number of clusters (Gaussian components) to output
+   * @param mean initial cluster centers for iterative refinement
+   * @param ns number of input samples
+   * @param sample input samples; will be sorted in ascending order during use
+   */
+  public void cluster(int n, double center[], int ns, double sample[])
+  {
+    // Record input parameters.
+    setCenters(n, center);
+    setSamples(ns, sample);
+    // Initialize EM iterations.
+    initEM();
+    // Perform EM iterations until convergence.
+    final double thresh = 1.0e-6;
+    double oldmsr = Double.MAX_VALUE;
+    for (int it=1;; ++it) {
+      // one EM iteration
+      expectation();
+      maximization();
+      // Check for convergence.
+      final double msr = measure();
+      final double dif = Math.abs(msr - oldmsr);
+      final double err = dif / (1.0 + oldmsr);
+      oldmsr = msr;
+      if (err < thresh) { break; }
+    }
+    // Compute cluster weights.
+    computeWeights();
+
+    // diagnostic messages
+    printMog("JAVA-COMPUTED", n, weight, mean, var);
+    BaseTestCase.println("msr = (" + oldmsr + ")");
+  }
+
+  /**
+   * Compute an initial configuration of cluster centers uniformly spaced
+   * over the range of the input samples, for subsequent iterative refinement.
+   * @param n number of clusters to output
+   * @param center initial uniform cluster centers to compute
+   * @param ns number of input samples
+   * @param sample array of input samples
+   */
+  public static void uniform(int n, double center[], int ns, double sample[])
+  {
+    double min_x = Double.MAX_VALUE, max_x = -Double.MAX_VALUE;
+    for (int i=0; i<ns; ++i) {
+      final double x = sample[i];
+      if (min_x > x) { min_x = x; }
+      if (max_x < x) { max_x = x; }
+    }
+
+    final double length = max_x - min_x;
+    final double increment = length / n;
+    center[0] = increment / 2;
+    for (int i=1; i<n; ++i) { center[i] = center[i-1] + increment; }
+  }
+
+  /**
+   * Compute an initial configuration of cluster centers uniformly distributed
+   * over the range of the input samples, for subsequent iterative refinement.
+   * @param n number of clusters to output
+   * @param center initial uniform cluster centers to compute
+   * @param ns number of input samples
+   * @param sample array of input samples
+   * @param rng random number generator
+   */
+  public static void random(int n, double center[], int ns, double sample[],
+                            Random rng)
+  {
+    double min_x = Double.MAX_VALUE, max_x = -Double.MAX_VALUE;
+    for (int i=0; i<ns; ++i) {
+      final double x = sample[i];
+      if (min_x > x) { min_x = x; }
+      if (max_x < x) { max_x = x; }
+    }
+
+    final double length = max_x - min_x;
+    for (int i=0; i<n; ++i) {
+      final double r = rng.nextDouble();
+      final double x = min_x + r * length;
+      center[i] = x;
+    }
+  }
+
+  /** Initialize cluster centers for EM iterations. */
+  void setCenters(int n, double center[])
+  {
+    if (1 <= n && n <= max_n) {
+      this.n = n;
+      System.arraycopy(center, 0, mean, 0, n);
+    }
+    else {
+      final String msg =
+        "Number of Gaussian components (" + n + ") not in [1, " + max_n + "].";
+      throw new IllegalArgumentException(msg);
+    }
+  }
+
+  /** Specify the input samples to work with. */
+  void setSamples(int ns, double sample[])
+  {
+    final int min_sample_size = n * min_sample_size_per_cluster;
+    if (ns >= min_sample_size) {
+      this.ns = ns;
+      this.sample = sample;
+    }
+    else {
+      final String msg =
+        "Insufficient sample size (" + ns + " < " + min_sample_size + ").";
+      throw new IllegalArgumentException(msg);
+    }
+  }
+
+  /** Initialize the EM (expectation-maximization) iterations. */
+  void initEM()
+  {
+    // Sort the input samples in ascending order.
+    Arrays.sort(sample, 0, ns);
+    // Sort the initial cluster centers in ascending order.
+    Arrays.sort(mean, 0, n);
+    // Initialize the cluster brackets.
+    maximization();
+  }
+
+  /** (Re-)compute cluster centers while holding cluster brackets fixed. */
+  void expectation()
+  {
+    // Remove empty clusters.
+    for (int i=0, j=1;;) {
+      // Examine the value at the current location.
+      final int bi = bracket[i];
+      // Locate the next larger value.
+      for (; j<n; ++j) {
+        final int bj = bracket[j];
+        if (bi < bj) {
+          // Move the larger value up to be adjacent to current value.
+          bracket[i+1] = bj;
+          // Advance loop variables.
+          ++i;  ++j;  break;
+        }
+      }
+      // Check for loop termination.
+      if (j >= n) { n = i+1;  break; }
+    }
+
+    // Compute cluster parameters.
+    for (int i=0; i<n; ++i) {
+      final int ini = bracket[i];
+      final int lim = bracket[i+1];
+      final int nb = (lim - ini);
+      // Computer cluster mean.
+      double sum = 0.0;
+      for (int j=ini; j<lim; ++j) {
+        final double x = sample[j];
+        sum += x;
+      }
+      final double m = (sum / nb);
+      mean[i] = m;
+      // Compute cluster variance.
+      sum = 0.0;
+      for (int j=ini; j<lim; ++j) {
+        final double x = sample[j];
+        final double d = x - m;
+        sum += d * d;
+      }
+      final double v = ((nb > 1) ? (sum / (nb-1)) : 0.0);
+      var[i] = v;
+    }
+  }
+
+  /** (Re-)compute cluster brackets while holding cluster centers fixed. */
+  void maximization()
+  {
+    bracket[0] = 0;
+    for (int i=1; i<n; ++i) {
+      final double mlo = mean[i-1];
+      final double mhi = mean[i];
+      // Compute the dividing point between clusters (i-1) and (i).
+      int lo = bracket[i-1], hi = ns;
+      while (lo < (hi-1)) {
+        final int mid = (lo + hi) >> 1;
+        final double sam = sample[mid];
+        final double dlo = Math.abs(sam - mlo);
+        final double dhi = Math.abs(mhi - sam);
+        if (dlo < dhi) { lo = mid; } else { hi = mid; }
+      }
+      bracket[i] = hi;
+    }
+    bracket[n] = ns;
+  }
+
+  /** Compute a measure of total quantization error. */
+  double measure()
+  {
+    double sum = 0.0;
+    for (int i=0; i<n; ++i) {
+      final int ini = bracket[i];
+      final int lim = bracket[i+1];
+      final int nb = lim - ini;
+      final double v = var[i];
+      sum += v * (nb-1);
+    }
+    sum /= ns;
+    return sum;
+  }
+
+  /** Compute cluster weights. */
+  void computeWeights()
+  {
+    for (int i=0; i<n; ++i) {
+      final int ini = bracket[i];
+      final int lim = bracket[i+1];
+      final int siz = lim - ini;
+      final double wt = ((ns > 0) ? ((double) siz / (double) ns) : 0.0);
+      weight[i] = wt;
+    }
+  }
+
+  /** Print out the clustering configuration. */
+  void printMog(String label, int n, double weight[], double mean[], double var[])
+  {
+    BaseTestCase.println(label + ": n = " + n);
+    for (int i=0; i<n; ++i) {
+      BaseTestCase.println("(w, m, v) = (" +
+              weight[i] + ", " + mean[i] + ", " + var[i] + ")");
+    }
+  }
+
+  /** maximum number of Gaussian components */
+  public final static int max_n = 6;
+
+  /** actual number of Gaussian components */
+  public int n = 0;
+  /** weights associated with the Gaussian components */
+  public final double weight[] = new double[max_n];
+  /** mean parameters for the Gaussian components */
+  public final double mean[] = new double[max_n];
+  /** variance parameters for the Gaussian components */
+  public final double var[] = new double[max_n];
+
+  /** cluster brackets on the input samples */
+  protected int bracket[] = new int[max_n+1];
+  /** number of input samples */
+  protected int ns = 0;
+  /** array of input samples */
+  protected double sample[] = null;
+
+  /** minimum sample size per output cluster */
+  public static final int min_sample_size_per_cluster = 32;
+}

Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMog.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMogSQL.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMogSQL.java?rev=759181&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMogSQL.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMogSQL.java
Fri Mar 27 15:45:59 2009
@@ -0,0 +1,316 @@
+/*
+   Derby - Class org.apache.derbyTesting.functionTests.tests.memorydb.ClusMogSQL
+
+   Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License.
+ */
+
+package org.apache.derbyTesting.functionTests.tests.memorydb;
+
+import java.util.Arrays;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import org.apache.derbyTesting.junit.BaseTestCase;
+
+/**
+ * Simple utility to compute/recover the parameters of a mixture-of-Gaussian
+ * distribution from independent samples, using SQL.
+ */
+public class ClusMogSQL extends ClusMog
+{
+  /** constructor */
+  public ClusMogSQL(Connection conn) { this.conn = conn; }
+
+  /** Set unique ID for this object. */
+  public void setUnique(int uniq)
+  {
+    this.uniq = (uniq & 0x7fffffff);
+  }
+
+  /**
+   * Compute/recover the parameters of a mixture-of-Gaussian distribution
+   * from given independent samples, using SQL.
+   * @param n number of clusters (Gaussian components) to output
+   * @param center initial cluster centers for iterative refinement
+   * @param ns number of input samples
+   * @param sample input samples; will be sorted in ascending order during use
+   */
+  public void clusterSQL(int n, double center[], int ns, double sample[])
+  throws SQLException
+  {
+    // Record input parameters.
+    setCenters(n, center);
+    setSamples(ns, sample);
+    // Initialize EM iterations.
+    init();
+    // Perform EM iterations until convergence.
+    final double thresh = 1.0e-6;
+    double oldmsr = Double.MAX_VALUE;
+    for (int it=1;; ++it) {
+      // one EM iteration
+      final double msr = expect();
+      maximize();
+      // Check for convergence.
+      final double dif = Math.abs(msr - oldmsr);
+      final double err = dif / (1.0 + oldmsr);
+      oldmsr = msr;
+      if (err < thresh) { break; }
+    }
+    // Download the cluster configuration.
+    download();
+    // Clean up working tables after use.
+    cleanup();
+
+    // diagnostic messages
+    printMog("SQL-COMPUTED", n, weight, mean, var);
+    BaseTestCase.println("msr = (" + oldmsr + ")");
+  }
+
+  /** Initialize the EM (expectation-maximization) iterations. */
+  void init() throws SQLException
+  {
+    // Sort the input samples in ascending order.
+    Arrays.sort(sample, 0, ns);
+    // Sort the initial cluster centers in ascending order.
+    Arrays.sort(mean, 0, n);
+
+    // working table names
+    final String clusterN = "cluster" + uniq;
+    final String sampleN = "sample" + uniq;
+
+    // Initialize database tables.
+    PreparedStatement pstmt = null;
+    Statement stmt = conn.createStatement();
+    try {
+      stmt.executeUpdate("CREATE TABLE " + clusterN + "(weight double, mean double, var double,
bucket int PRIMARY KEY)");
+      stmt.executeUpdate("CREATE TABLE " + sampleN + "(value double, id int PRIMARY KEY,
bucket int)");
+
+      pstmt = conn.prepareStatement("INSERT INTO " + sampleN + "(value, id) VALUES (?, ?)");
+      for (int i=0; i<ns; ++i) {
+        final double x = sample[i];
+        pstmt.setDouble(1, x);
+        pstmt.setInt(2, i);
+        pstmt.executeUpdate();
+      }
+      pstmt.close();
+      pstmt = conn.prepareStatement("INSERT INTO " + clusterN + "(mean, bucket) VALUES (?,
?)");
+      for (int i=0; i<n; ++i) {
+        final double x = mean[i];
+        pstmt.setDouble(1, x);
+        pstmt.setInt(2, i);
+        pstmt.executeUpdate();
+      }
+    }
+    finally {
+      if (stmt != null) { stmt.close();  stmt = null; }
+      if (pstmt != null) { pstmt.close();  pstmt = null; }
+    }
+
+    // Initialize sample-to-cluster assignment.
+    maximize();
+  }
+
+  /**
+   * (Re-)compute cluster centers while holding sample-to-cluster assignment fixed.
+   * @return mean square error of resulting clustering configuration
+   * @throws SQLException
+   */
+  double expect() throws SQLException
+  {
+    // working table names
+    final String clusterN = "cluster" + uniq;
+    final String sampleN = "sample" + uniq;
+    final String mm = "mm" + uniq;
+    final String vv = "vv" + uniq;
+    final String ee = "ee" + uniq;
+
+    double msr = Double.MAX_VALUE;
+    Statement stmt = null;
+    ResultSet rset = null;
+    try {
+      stmt = conn.createStatement();
+
+      stmt.executeUpdate("CREATE TABLE " + mm + "(bucket int PRIMARY KEY, mean double)");
+      stmt.executeUpdate("CREATE TABLE " + vv + "(bucket int PRIMARY KEY, var double)");
+      stmt.executeUpdate("CREATE TABLE " + ee + "(bucket int PRIMARY KEY, err double, size
int)");
+
+      stmt.executeUpdate("INSERT INTO " + mm + "(bucket, mean) \n" +
+                         "SELECT bucket, avg(value) \n" +
+                         "  FROM " + sampleN + " \n" +
+                         " GROUP BY bucket \n");
+
+      stmt.executeUpdate("INSERT INTO " + ee + "(bucket, err, size) \n" +
+                         "SELECT S.bucket, sum((S.value - M.mean) * (S.value - M.mean)),
count(*) \n" +
+                         "  FROM " + sampleN + " S JOIN " + mm + " M ON S.bucket = M.bucket
\n" +
+                         " GROUP BY S.bucket \n");
+
+      stmt.executeUpdate("INSERT INTO " + vv + "(bucket, var) \n" +
+                         "SELECT bucket, \n" +
+                         "       CASE WHEN (size > 1) THEN (err / (size - 1)) ELSE 0.0
END \n" +
+                         "  FROM " + ee + " \n");
+
+      stmt.executeUpdate("DELETE FROM " + clusterN);
+
+      stmt.executeUpdate("INSERT INTO " + clusterN + "(mean, var, bucket) \n" +
+                         "SELECT M.mean, V.var, V.bucket \n" +
+                         "  FROM " + mm + " M JOIN " + vv + " V ON M.bucket = V.bucket \n");
+
+      rset = stmt.executeQuery("SELECT (sum(err) / sum(size)) AS measure FROM " + ee);
+      while (rset.next()) { msr = rset.getDouble(1); }
+
+      stmt.executeUpdate("DROP TABLE " + mm);
+      stmt.executeUpdate("DROP TABLE " + vv);
+      stmt.executeUpdate("DROP TABLE " + ee);
+    }
+    finally {
+      if (rset != null) { rset.close();  rset = null; }
+      if (stmt != null) { stmt.close();  stmt = null; }
+    }
+    return msr;
+  }
+
+  /**
+   * (Re-)compute sample-to-cluster assignment while holding cluster centers fixed.
+   * @throws SQLException
+   */
+  void maximize() throws SQLException
+  {
+    // working table names
+    final String clusterN = "cluster" + uniq;
+    final String sampleN = "sample" + uniq;
+    final String gg = "gg" + uniq;
+    final String jj = "jj" + uniq;
+
+    Statement stmt = null;
+    try {
+      stmt = conn.createStatement();
+
+      stmt.executeUpdate("CREATE TABLE " + gg + "(id int PRIMARY KEY, diff double)");
+      stmt.executeUpdate("CREATE TABLE " + jj + "(value double, id int, diff double, bucket
int)");
+
+      stmt.executeUpdate("INSERT INTO " + gg + "(id, diff) \n" +
+                         "SELECT S.id, min(abs(S.value - C.mean)) \n" +
+                         "  FROM " + sampleN + " S, " + clusterN + " C \n" +
+                         " GROUP BY S.id \n");
+
+      stmt.executeUpdate("INSERT INTO " + jj + "(value, id, diff, bucket) \n" +
+                         "SELECT S.value, S.id, abs(S.value - C.mean), C.bucket \n" +
+                         "  FROM " + sampleN + " S, " + clusterN + " C \n");
+
+      stmt.executeUpdate("DELETE FROM " + sampleN);
+
+      stmt.executeUpdate("INSERT INTO " + sampleN + "(value, id, bucket) \n" +
+                         "SELECT J.value, J.id, min(J.bucket) \n" +
+                         "  FROM " + jj + " J \n" +
+                         "  JOIN " + gg + " G \n" +
+                         "    ON J.id   = G.id \n" +
+                         "   AND J.diff = G.diff \n" +
+                         " GROUP BY J.id, J.value \n");
+
+      stmt.executeUpdate("DROP TABLE " + gg);
+      stmt.executeUpdate("DROP TABLE " + jj);
+    }
+    finally {
+      if (stmt != null) { stmt.close(); }
+    }
+  }
+
+  /**
+   * Download the computed cluster configuration.
+   * @throws SQLException
+   */
+  void download() throws SQLException
+  {
+    // working table names
+    final String clusterN = "cluster" + uniq;
+    final String sampleN = "sample" + uniq;
+    final String ww = "ww" + uniq;
+    final String cc = "cc" + uniq;
+
+    Statement stmt = null;
+    ResultSet rset = null;
+    try {
+      stmt = conn.createStatement();
+
+      stmt.executeUpdate("CREATE TABLE " + ww + "(bucket int PRIMARY KEY, size int)");
+      stmt.executeUpdate("CREATE TABLE " + cc + "(weight double, mean double, var double,
bucket int PRIMARY KEY)");
+
+      stmt.executeUpdate("INSERT INTO " + ww + "(bucket, size) \n" +
+                         "SELECT bucket, count(*) \n" +
+                         "  FROM " + sampleN + " \n" +
+                         " GROUP BY bucket \n");
+
+      stmt.executeUpdate("INSERT INTO " + cc + "(weight, mean, var, bucket) \n" +
+                         "SELECT (CAST(W.size AS double) / (SELECT sum(size) FROM " + ww
+ ")), C.mean, C.var, C.bucket \n" +
+                         "  FROM " + clusterN + " C JOIN " + ww + " W ON C.bucket = W.bucket
\n");
+
+      stmt.executeUpdate("DELETE FROM " + clusterN);
+
+      stmt.executeUpdate("INSERT INTO " + clusterN + "(weight, mean, var, bucket) \n" +
+                         "SELECT weight, mean, var, bucket FROM " + cc + " \n");
+
+      stmt.executeUpdate("DROP TABLE " + ww);
+      stmt.executeUpdate("DROP TABLE " + cc);
+
+      rset = stmt.executeQuery("SELECT weight, mean, var FROM " + clusterN + " ORDER BY mean");
+      n = 0;
+      while (rset.next()) {
+        final double w = rset.getDouble(1);
+        final double m = rset.getDouble(2);
+        final double v = rset.getDouble(3);
+        weight[n] = w;
+        mean[n] = m;
+        var[n] = v;
+        ++n;
+      }
+    }
+    finally {
+      if (rset != null) { rset.close();  rset = null; }
+      if (stmt != null) { stmt.close();  stmt = null; }
+    }
+  }
+
+  /**
+   * Clean up working tables after use.
+   * @throws SQLException
+   */
+  void cleanup() throws SQLException
+  {
+    // working table names
+    final String clusterN = "cluster" + uniq;
+    final String sampleN = "sample" + uniq;
+
+    Statement stmt = null;
+    try {
+      stmt = conn.createStatement();
+      stmt.executeUpdate("DROP TABLE " + sampleN);
+      stmt.executeUpdate("DROP TABLE " + clusterN);
+    }
+    finally {
+      if (stmt != null) { stmt.close();  stmt = null; }
+    }
+  }
+
+  /** database connection */
+  Connection conn;
+
+  /** unique ID for generating working table names */
+  int uniq = 0;
+}

Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMogSQL.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/GenMog.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/GenMog.java?rev=759181&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/GenMog.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/GenMog.java
Fri Mar 27 15:45:59 2009
@@ -0,0 +1,91 @@
+/*
+   Derby - Class org.apache.derbyTesting.functionTests.tests.memorydb.GenMog
+
+   Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License.
+ */
+
+package org.apache.derbyTesting.functionTests.tests.memorydb;
+
+import java.util.Random;
+import org.apache.derbyTesting.junit.BaseTestCase;
+
+/**
+ * Simple utility to generate a mixture-of-Gaussian configuration.
+ */
+public class GenMog
+{
+  /** default constructor */
+  public GenMog() { rng = new Random(System.currentTimeMillis()); }
+
+  /** constructor with specified RNG */
+  public GenMog(Random rng) { this.rng = rng; }
+
+  /** Generate a mixture-of-Gaussian configuration. */
+  public void generate()
+  {
+    // number of Gaussian components
+    n = 1 + rng.nextInt(max_n);
+
+    // weights associated with the Gaussian components
+    double sum = 0.0;
+    for (int i=0; i<n; ++i) {
+      double w;
+      do { w = rng.nextDouble(); } while (w <= 0.0);
+      weight[i] = w;
+      sum += w;
+    }
+    for (int i=0; i<n; ++i) { weight[i] /= sum; }
+
+    // (mean, var) parameters for the Gaussian components
+    double oldm = 0.0, olds = 0.0;
+    for (int i=0; i<n; ++i) {
+      final double s = min_s + rng.nextDouble() * (max_s - min_s);
+      final double m = oldm + 2.0 * olds + 2.0 * s;
+      mean[i] = m;
+      var[i] = s * s;
+      oldm = m;
+      olds = s;
+    }
+
+    // diagnostic messages
+    BaseTestCase.println("GENERATED: n = " + n);
+    for (int i=0; i<n; ++i) {
+      BaseTestCase.println("(w, m, v) = (" +
+              weight[i] + ", " + mean[i] + ", " + var[i] + ")");
+    }
+  }
+
+  /** random number generator in use */
+  public Random rng;
+
+  /** maximum number of Gaussian components */
+  public final static int max_n = 6;
+  /** minimum value for the standard deviation parameter */
+  public final static double min_s = 1.0;
+  /** maximum value for the standard deviation parameter */
+  public final static double max_s = 6.0;
+
+  /** actual number of Gaussian components */
+  public int n;
+  /** weights associated with the Gaussian components */
+  public final double weight[] = new double[max_n];
+  /** mean parameters for the Gaussian components */
+  public final double mean[] = new double[max_n];
+  /** variance parameters for the Gaussian components */
+  public final double var[] = new double[max_n];
+}

Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/GenMog.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/MogTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/MogTest.java?rev=759181&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/MogTest.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/MogTest.java
Fri Mar 27 15:45:59 2009
@@ -0,0 +1,194 @@
+/*
+   Derby - Class org.apache.derbyTesting.functionTests.tests.memorydb.MogTest
+
+   Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License.
+ */
+
+package org.apache.derbyTesting.functionTests.tests.memorydb;
+
+import junit.framework.Test;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import org.apache.derbyTesting.junit.BaseJDBCTestCase;
+import org.apache.derbyTesting.junit.TestConfiguration;
+
+/**
+ * Test consistency among (GenMog), (SampMog), and (ClusMog).
+ */
+public class MogTest extends BaseJDBCTestCase {
+
+  public MogTest(String name) { super(name); }
+	
+  public static Test suite() {
+    return TestConfiguration.defaultSuite(MogTest.class);
+  }
+
+  /** Dispose of objects after testing. */
+  protected void tearDown() throws Exception
+  {
+    super.tearDown();
+  }
+
+  /**
+   * Calculates by using the default directory/disk storage back end.
+   *
+   * @throws SQLException if the test fails
+   */
+  public void testClusMogOnDisk()
+          throws SQLException {
+      long start = System.currentTimeMillis();
+      doTestClusMog(getConnection());
+      println("duration-on-disk: " + (System.currentTimeMillis() - start));
+  }
+
+  /**
+   * Calculates by using the in-memory storage back end.
+   *
+   * @throws SQLException if the test fails
+   */
+  public void testClusMogInMemory()
+          throws SQLException {
+      long start = System.currentTimeMillis();
+      // Close the connection explicitly here, since the test framework won't
+      // do it for us when we create it manually.
+      Connection conn = obtainConnection();
+      try {
+          doTestClusMog(conn);
+      } finally {
+          try {
+              conn.rollback();
+              conn.close();
+          } catch (SQLException sqle) {
+              // Ignore exceptions during close.
+          }
+      }
+      println("duration-in-memory: " + (System.currentTimeMillis() - start));
+  }
+
+  /**
+   * Test consistency between (ClusMog) and (ClusMogSQL).
+   * @throws SQLException if something goes wrong
+   */
+  public void doTestClusMog(Connection conn)
+          throws SQLException {
+
+    // Initialize test objects.
+    long _seed = System.currentTimeMillis();
+    java.util.Random rng = new java.util.Random(_seed);
+    /** MOG generator being tested */
+    GenMog genMog = new GenMog(rng);
+    /** MOG sampler being tested */
+    SampMog sampMog = new SampMog(rng);
+    /** clustering object being tested */
+    ClusMog clusMog = new ClusMog();
+    /** clustering object being tested */
+    ClusMogSQL clusMogSql = new ClusMogSQL(conn);
+    clusMogSql.setUnique(rng.nextInt());
+
+    println(getName() + " using random seed: " + _seed);
+    final int max_ns = 10 * ClusMog.min_sample_size_per_cluster * GenMog.max_n;
+    final double sample[] = new double[max_ns];
+    final double center[] = new double[GenMog.max_n];
+    final int niter = 1;
+    for (int i=niter; i>0; --i) {
+      // Generate a MOG configuration.
+      genMog.generate();
+      // Compute a sample size.
+      final int min_ns = ClusMog.min_sample_size_per_cluster * genMog.n;
+      final int ns = min_ns + rng.nextInt(max_ns - min_ns);
+      println("ns = " + ns);
+      // Generate samples from the MOG distribution.
+      sampMog.set(genMog.n, genMog.weight, genMog.mean, genMog.var);
+      sampMog.generate(ns, sample);
+
+      // Produce an initial cluster center configuration.
+      ClusMog.uniform(genMog.n, center, ns, sample);
+      // Cluster the samples to recover the MOG configuration.
+      clusMog.cluster(genMog.n, center, ns, sample);
+      // Cluster the samples again, using SQL.
+      clusMogSql.clusterSQL(genMog.n, center, ns, sample);
+      // Compare the computed MOG configurations.
+      assertEquals(clusMog.n, clusMogSql.n);
+      compare(clusMog.n, clusMog.weight, clusMogSql.weight);
+      compare(clusMog.n, clusMog.mean, clusMogSql.mean);
+      compare(clusMog.n, clusMog.var, clusMogSql.var);
+
+      // Produce another initial cluster center configuration.
+      ClusMog.random(genMog.n, center, ns, sample, rng);
+      // Cluster the samples to recover the MOG configuration.
+      clusMog.cluster(genMog.n, center, ns, sample);
+      // Cluster the samples again, using SQL.
+      clusMogSql.clusterSQL(genMog.n, center, ns, sample);
+      // Compare the computed MOG configurations.
+      assertEquals(clusMog.n, clusMogSql.n);
+      compare(clusMog.n, clusMog.weight, clusMogSql.weight);
+      compare(clusMog.n, clusMog.mean, clusMogSql.mean);
+      compare(clusMog.n, clusMog.var, clusMogSql.var);
+    }
+  }
+
+  /** Compare two floating-point arrays, with tolerance. */
+  private void compare(int n, double ones[], double oths[])
+  {
+    final double thresh = 1.0e-6;
+    for (int i=0; i<n; ++i) {
+      final double one = ones[i];
+      final double oth = oths[i];
+      final double dif = Math.abs(one - oth);
+      final double err = dif / (1.0 + Math.abs(one));
+      assertTrue(err < thresh);
+    }
+  }
+
+  /**
+   * Obtains a connection to an in-memory database.
+   *
+   * @return A connection to an in-memory database.
+   * @throws SQLException if obtaining the connection fails
+   */
+  private Connection obtainConnection()
+        throws SQLException {
+    try {
+        if (usingDerbyNetClient()) {
+            Class.forName("org.apache.derby.jdbc.ClientDriver");
+        } else {
+            Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+        }
+    } catch (Exception e) {
+        SQLException sqle =  new SQLException(e.getMessage());
+        sqle.initCause(e);
+        throw sqle;
+    }
+    StringBuffer sb = new StringBuffer("jdbc:derby:");
+    if (usingEmbedded()) {
+        sb.append("memory:");
+    } else {
+        // This is a hack. Change this when proper support for the in-memory
+        // back end has been implemented.
+        sb.append("//");
+        sb.append(TestConfiguration.getCurrent().getHostName());
+        sb.append(':');
+        sb.append(TestConfiguration.getCurrent().getPort());
+        sb.append('/');
+        sb.append("memory:");
+    }
+    sb.append("MogTestDb;create=true");
+    return DriverManager.getConnection(sb.toString());
+  }
+}

Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/MogTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/SampMog.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/SampMog.java?rev=759181&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/SampMog.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/SampMog.java
Fri Mar 27 15:45:59 2009
@@ -0,0 +1,168 @@
+/*
+   Derby - Class org.apache.derbyTesting.functionTests.tests.memorydb.SampMog
+
+   Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License.
+ */
+
+package org.apache.derbyTesting.functionTests.tests.memorydb;
+
+import java.util.Arrays;
+import java.util.Random;
+
+/**
+ * Simple utility to generate samples from a mixture-of-Gaussian distribution.
+ */
+public class SampMog
+{
+  /** default constructor */
+  public SampMog() { rng = new Random(System.currentTimeMillis()); }
+
+  /** constructor with specified RNG */
+  public SampMog(Random rng) { this.rng = rng; }
+
+  /**
+   * Generate samples from the specified mixture-of-Gaussian distribution.
+   * @param ns number of samples to generate
+   * @param sample output array of generated samples
+   */
+  public void generate(int ns, double sample[])
+  {
+    // First, generate standard normal samples, drawn from N(0, 1).
+    for (int i=0; i<ns; ++i) {
+      final double r = rng.nextGaussian();
+      sample[i] = r;
+    }
+    // Then, transform the samples to conform to the Gaussian components
+    // according to their weights.
+    for (int i=0; i<ns; ++i) {
+      // Pick a Gaussian component, represented by (idx).
+      final double w = rng.nextDouble();
+      int idx = Arrays.binarySearch(cumulative, w);
+      if (idx < 0) { idx = -(idx + 1); }
+      if (idx >= n) { idx = n-1; }
+      final double m = mean[idx];
+      final double s = stdv[idx];
+      // Transform the sample to conform to the Gaussian component.
+      final double r = sample[i];
+      final double x = m + s * r;
+      sample[i] = x;
+    }
+  }
+
+  /** Get the maximum number of Gaussian components. */
+  public static int getMaxNumber() { return max_n; }
+
+  /** Get the number of Gaussian components. */
+  public int getNumber() { return n; }
+
+  /** Specify the mixture-of-Gaussian configuration. */
+  public void set(int n, double wts[], double mm[], double vv[])
+  {
+    setNumber(n);
+    setWeights(wts);
+    setMeans(mm);
+    setVars(vv);
+  }
+
+  /** Set the number of Gaussian components. */
+  public void setNumber(int n)
+  {
+    if (1 <= n && n <= max_n) {
+      this.n = n;
+    }
+    else {
+      final String msg =
+        "Number of Gaussian components (" + n + ") not in [1, " + max_n + "].";
+      throw new IllegalArgumentException(msg);
+    }
+  }
+
+  /** Specify the weights for the Gaussian components. */
+  public void setWeights(double wts[])
+  {
+    // Copy weights to internal state array.
+    System.arraycopy(wts, 0, weight, 0, n);
+
+    // Normalize the weights to sum to 1.
+    IllegalArgumentException ex = null;
+    double sum = 0.0;
+    for (int i=0; i<n; ++i) {
+      final double wt = weight[i];
+      if (wt > 0.0) {
+        sum += wt;
+      }
+      else {
+        if (ex == null) {
+          final String msg = "Invalid weight (" + wt + ").";
+          ex = new IllegalArgumentException(msg);
+        }
+      }
+    }
+    if (sum > 0.0) {
+      for (int i=0; i<n; ++i) { weight[i] /= sum; }
+    }
+    else {
+      if (ex == null) {
+        final String msg = "Invalid total weight (" + sum + ").";
+        ex = new IllegalArgumentException(msg);
+      }
+    }
+    if (ex != null) { throw ex; }
+
+    // Compute cumulative weights.
+    cumulative[0] = weight[0];
+    for (int i=1; i<n; ++i) { cumulative[i] = Math.min(cumulative[i-1] + weight[i], 1.0);
}
+    for (int i=n; i<max_n; ++i) { cumulative[i] = 1.0; }
+  }
+
+  /** Specify the mean parameters for the Gaussian components. */
+  public void setMeans(double mm[])
+  {
+    System.arraycopy(mm, 0, mean, 0, n);
+  }
+
+  /** Specify the variance parameters for the Gaussian components. */
+  public void setVars(double vv[])
+  {
+    for (int i=0; i<n; ++i) {
+      final double v = Math.abs(vv[i]);
+      var[i] = v;
+      stdv[i] = Math.sqrt(v);
+    }
+  }
+
+  /** random number generator in use */
+  public Random rng;
+
+  /** maximum number of Gaussian components */
+  public final static int max_n = 6;
+
+  /** actual number of Gaussian components */
+  int n = 3;
+  /** weights associated with the Gaussian components */
+  final double weight[] = new double[max_n];
+  /** mean parameters for the Gaussian components */
+  final double mean[] = new double[max_n];
+  /** variance parameters for the Gaussian components */
+  final double var[] = new double[max_n];
+
+  /** cumulative weights, for sample generation */
+  final double cumulative[] = new double[max_n];
+  /** standard deviation parameters for the Gaussian components */
+  final double stdv[] = new double[max_n];
+}

Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/SampMog.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/_Suite.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/_Suite.java?rev=759181&r1=759180&r2=759181&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/_Suite.java
(original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/_Suite.java
Fri Mar 27 15:45:59 2009
@@ -22,6 +22,7 @@
 package org.apache.derbyTesting.functionTests.tests.memorydb;
 
 import org.apache.derbyTesting.junit.BaseTestCase;
+import org.apache.derbyTesting.junit.JDBC;
 import junit.framework.Test;
 import junit.framework.TestSuite;
 
@@ -42,8 +43,11 @@
     public static Test suite() {
 
         TestSuite suite = new TestSuite("In-memory db test suite");
-        suite.addTest(BasicInMemoryDbTest.suite());
-
+        // Tests are not yet compatible with JSR169 / JavaME
+        if (!JDBC.vmSupportsJSR169()) {
+            suite.addTest(BasicInMemoryDbTest.suite());
+            suite.addTest(MogTest.suite());
+        }
         return suite;
     }
 }



Mime
View raw message