Author: kristwaa
Date: Fri Mar 27 15:45:59 2009
New Revision: 759181
URL: http://svn.apache.org/viewvc?rev=759181&view=rev
Log:
DERBY-4085: Improve testing of the in-memory back end.
Adds a functional test (mixture-of-Gaussian distribution).
The calculations are carried out both in Java and though SQL, then compared.
The tests are not yet compatible with JSR 169 / JavaME.
Patch contributed by Cheng Che Chen (chatom).
Minor modifications by Kristian Waagan.
Patch file: derby-4085-2a-mog_func_test.diff
Added:
db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMog.java
(with props)
db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMogSQL.java
(with props)
db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/GenMog.java
(with props)
db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/MogTest.java
(with props)
db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/SampMog.java
(with props)
Modified:
db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/_Suite.java
Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMog.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMog.java?rev=759181&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMog.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMog.java
Fri Mar 27 15:45:59 2009
@@ -0,0 +1,289 @@
+/*
+ Derby - Class org.apache.derbyTesting.functionTests.tests.memorydb.ClusMog
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ */
+
+package org.apache.derbyTesting.functionTests.tests.memorydb;
+
+import java.util.Arrays;
+import java.util.Random;
+import org.apache.derbyTesting.junit.BaseTestCase;
+
+/**
+ * Simple utility to compute/recover the parameters of a mixture-of-Gaussian
+ * distribution from independent samples.
+ */
+public class ClusMog
+{
+ /** default constructor */
+ public ClusMog() {}
+
+ /**
+ * Compute/recover the parameters of a mixture-of-Gaussian distribution
+ * from given independent samples.
+ * @param n number of clusters (Gaussian components) to output
+ * @param mean initial cluster centers for iterative refinement
+ * @param ns number of input samples
+ * @param sample input samples; will be sorted in ascending order during use
+ */
+ public void cluster(int n, double center[], int ns, double sample[])
+ {
+ // Record input parameters.
+ setCenters(n, center);
+ setSamples(ns, sample);
+ // Initialize EM iterations.
+ initEM();
+ // Perform EM iterations until convergence.
+ final double thresh = 1.0e-6;
+ double oldmsr = Double.MAX_VALUE;
+ for (int it=1;; ++it) {
+ // one EM iteration
+ expectation();
+ maximization();
+ // Check for convergence.
+ final double msr = measure();
+ final double dif = Math.abs(msr - oldmsr);
+ final double err = dif / (1.0 + oldmsr);
+ oldmsr = msr;
+ if (err < thresh) { break; }
+ }
+ // Compute cluster weights.
+ computeWeights();
+
+ // diagnostic messages
+ printMog("JAVA-COMPUTED", n, weight, mean, var);
+ BaseTestCase.println("msr = (" + oldmsr + ")");
+ }
+
+ /**
+ * Compute an initial configuration of cluster centers uniformly spaced
+ * over the range of the input samples, for subsequent iterative refinement.
+ * @param n number of clusters to output
+ * @param center initial uniform cluster centers to compute
+ * @param ns number of input samples
+ * @param sample array of input samples
+ */
+ public static void uniform(int n, double center[], int ns, double sample[])
+ {
+ double min_x = Double.MAX_VALUE, max_x = -Double.MAX_VALUE;
+ for (int i=0; i<ns; ++i) {
+ final double x = sample[i];
+ if (min_x > x) { min_x = x; }
+ if (max_x < x) { max_x = x; }
+ }
+
+ final double length = max_x - min_x;
+ final double increment = length / n;
+ center[0] = increment / 2;
+ for (int i=1; i<n; ++i) { center[i] = center[i-1] + increment; }
+ }
+
+ /**
+ * Compute an initial configuration of cluster centers uniformly distributed
+ * over the range of the input samples, for subsequent iterative refinement.
+ * @param n number of clusters to output
+ * @param center initial uniform cluster centers to compute
+ * @param ns number of input samples
+ * @param sample array of input samples
+ * @param rng random number generator
+ */
+ public static void random(int n, double center[], int ns, double sample[],
+ Random rng)
+ {
+ double min_x = Double.MAX_VALUE, max_x = -Double.MAX_VALUE;
+ for (int i=0; i<ns; ++i) {
+ final double x = sample[i];
+ if (min_x > x) { min_x = x; }
+ if (max_x < x) { max_x = x; }
+ }
+
+ final double length = max_x - min_x;
+ for (int i=0; i<n; ++i) {
+ final double r = rng.nextDouble();
+ final double x = min_x + r * length;
+ center[i] = x;
+ }
+ }
+
+ /** Initialize cluster centers for EM iterations. */
+ void setCenters(int n, double center[])
+ {
+ if (1 <= n && n <= max_n) {
+ this.n = n;
+ System.arraycopy(center, 0, mean, 0, n);
+ }
+ else {
+ final String msg =
+ "Number of Gaussian components (" + n + ") not in [1, " + max_n + "].";
+ throw new IllegalArgumentException(msg);
+ }
+ }
+
+ /** Specify the input samples to work with. */
+ void setSamples(int ns, double sample[])
+ {
+ final int min_sample_size = n * min_sample_size_per_cluster;
+ if (ns >= min_sample_size) {
+ this.ns = ns;
+ this.sample = sample;
+ }
+ else {
+ final String msg =
+ "Insufficient sample size (" + ns + " < " + min_sample_size + ").";
+ throw new IllegalArgumentException(msg);
+ }
+ }
+
+ /** Initialize the EM (expectation-maximization) iterations. */
+ void initEM()
+ {
+ // Sort the input samples in ascending order.
+ Arrays.sort(sample, 0, ns);
+ // Sort the initial cluster centers in ascending order.
+ Arrays.sort(mean, 0, n);
+ // Initialize the cluster brackets.
+ maximization();
+ }
+
+ /** (Re-)compute cluster centers while holding cluster brackets fixed. */
+ void expectation()
+ {
+ // Remove empty clusters.
+ for (int i=0, j=1;;) {
+ // Examine the value at the current location.
+ final int bi = bracket[i];
+ // Locate the next larger value.
+ for (; j<n; ++j) {
+ final int bj = bracket[j];
+ if (bi < bj) {
+ // Move the larger value up to be adjacent to current value.
+ bracket[i+1] = bj;
+ // Advance loop variables.
+ ++i; ++j; break;
+ }
+ }
+ // Check for loop termination.
+ if (j >= n) { n = i+1; break; }
+ }
+
+ // Compute cluster parameters.
+ for (int i=0; i<n; ++i) {
+ final int ini = bracket[i];
+ final int lim = bracket[i+1];
+ final int nb = (lim - ini);
+ // Computer cluster mean.
+ double sum = 0.0;
+ for (int j=ini; j<lim; ++j) {
+ final double x = sample[j];
+ sum += x;
+ }
+ final double m = (sum / nb);
+ mean[i] = m;
+ // Compute cluster variance.
+ sum = 0.0;
+ for (int j=ini; j<lim; ++j) {
+ final double x = sample[j];
+ final double d = x - m;
+ sum += d * d;
+ }
+ final double v = ((nb > 1) ? (sum / (nb-1)) : 0.0);
+ var[i] = v;
+ }
+ }
+
+ /** (Re-)compute cluster brackets while holding cluster centers fixed. */
+ void maximization()
+ {
+ bracket[0] = 0;
+ for (int i=1; i<n; ++i) {
+ final double mlo = mean[i-1];
+ final double mhi = mean[i];
+ // Compute the dividing point between clusters (i-1) and (i).
+ int lo = bracket[i-1], hi = ns;
+ while (lo < (hi-1)) {
+ final int mid = (lo + hi) >> 1;
+ final double sam = sample[mid];
+ final double dlo = Math.abs(sam - mlo);
+ final double dhi = Math.abs(mhi - sam);
+ if (dlo < dhi) { lo = mid; } else { hi = mid; }
+ }
+ bracket[i] = hi;
+ }
+ bracket[n] = ns;
+ }
+
+ /** Compute a measure of total quantization error. */
+ double measure()
+ {
+ double sum = 0.0;
+ for (int i=0; i<n; ++i) {
+ final int ini = bracket[i];
+ final int lim = bracket[i+1];
+ final int nb = lim - ini;
+ final double v = var[i];
+ sum += v * (nb-1);
+ }
+ sum /= ns;
+ return sum;
+ }
+
+ /** Compute cluster weights. */
+ void computeWeights()
+ {
+ for (int i=0; i<n; ++i) {
+ final int ini = bracket[i];
+ final int lim = bracket[i+1];
+ final int siz = lim - ini;
+ final double wt = ((ns > 0) ? ((double) siz / (double) ns) : 0.0);
+ weight[i] = wt;
+ }
+ }
+
+ /** Print out the clustering configuration. */
+ void printMog(String label, int n, double weight[], double mean[], double var[])
+ {
+ BaseTestCase.println(label + ": n = " + n);
+ for (int i=0; i<n; ++i) {
+ BaseTestCase.println("(w, m, v) = (" +
+ weight[i] + ", " + mean[i] + ", " + var[i] + ")");
+ }
+ }
+
+ /** maximum number of Gaussian components */
+ public final static int max_n = 6;
+
+ /** actual number of Gaussian components */
+ public int n = 0;
+ /** weights associated with the Gaussian components */
+ public final double weight[] = new double[max_n];
+ /** mean parameters for the Gaussian components */
+ public final double mean[] = new double[max_n];
+ /** variance parameters for the Gaussian components */
+ public final double var[] = new double[max_n];
+
+ /** cluster brackets on the input samples */
+ protected int bracket[] = new int[max_n+1];
+ /** number of input samples */
+ protected int ns = 0;
+ /** array of input samples */
+ protected double sample[] = null;
+
+ /** minimum sample size per output cluster */
+ public static final int min_sample_size_per_cluster = 32;
+}
Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMog.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMogSQL.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMogSQL.java?rev=759181&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMogSQL.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMogSQL.java
Fri Mar 27 15:45:59 2009
@@ -0,0 +1,316 @@
+/*
+ Derby - Class org.apache.derbyTesting.functionTests.tests.memorydb.ClusMogSQL
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ */
+
+package org.apache.derbyTesting.functionTests.tests.memorydb;
+
+import java.util.Arrays;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import org.apache.derbyTesting.junit.BaseTestCase;
+
+/**
+ * Simple utility to compute/recover the parameters of a mixture-of-Gaussian
+ * distribution from independent samples, using SQL.
+ */
+public class ClusMogSQL extends ClusMog
+{
+ /** constructor */
+ public ClusMogSQL(Connection conn) { this.conn = conn; }
+
+ /** Set unique ID for this object. */
+ public void setUnique(int uniq)
+ {
+ this.uniq = (uniq & 0x7fffffff);
+ }
+
+ /**
+ * Compute/recover the parameters of a mixture-of-Gaussian distribution
+ * from given independent samples, using SQL.
+ * @param n number of clusters (Gaussian components) to output
+ * @param center initial cluster centers for iterative refinement
+ * @param ns number of input samples
+ * @param sample input samples; will be sorted in ascending order during use
+ */
+ public void clusterSQL(int n, double center[], int ns, double sample[])
+ throws SQLException
+ {
+ // Record input parameters.
+ setCenters(n, center);
+ setSamples(ns, sample);
+ // Initialize EM iterations.
+ init();
+ // Perform EM iterations until convergence.
+ final double thresh = 1.0e-6;
+ double oldmsr = Double.MAX_VALUE;
+ for (int it=1;; ++it) {
+ // one EM iteration
+ final double msr = expect();
+ maximize();
+ // Check for convergence.
+ final double dif = Math.abs(msr - oldmsr);
+ final double err = dif / (1.0 + oldmsr);
+ oldmsr = msr;
+ if (err < thresh) { break; }
+ }
+ // Download the cluster configuration.
+ download();
+ // Clean up working tables after use.
+ cleanup();
+
+ // diagnostic messages
+ printMog("SQL-COMPUTED", n, weight, mean, var);
+ BaseTestCase.println("msr = (" + oldmsr + ")");
+ }
+
+ /** Initialize the EM (expectation-maximization) iterations. */
+ void init() throws SQLException
+ {
+ // Sort the input samples in ascending order.
+ Arrays.sort(sample, 0, ns);
+ // Sort the initial cluster centers in ascending order.
+ Arrays.sort(mean, 0, n);
+
+ // working table names
+ final String clusterN = "cluster" + uniq;
+ final String sampleN = "sample" + uniq;
+
+ // Initialize database tables.
+ PreparedStatement pstmt = null;
+ Statement stmt = conn.createStatement();
+ try {
+ stmt.executeUpdate("CREATE TABLE " + clusterN + "(weight double, mean double, var double,
bucket int PRIMARY KEY)");
+ stmt.executeUpdate("CREATE TABLE " + sampleN + "(value double, id int PRIMARY KEY,
bucket int)");
+
+ pstmt = conn.prepareStatement("INSERT INTO " + sampleN + "(value, id) VALUES (?, ?)");
+ for (int i=0; i<ns; ++i) {
+ final double x = sample[i];
+ pstmt.setDouble(1, x);
+ pstmt.setInt(2, i);
+ pstmt.executeUpdate();
+ }
+ pstmt.close();
+ pstmt = conn.prepareStatement("INSERT INTO " + clusterN + "(mean, bucket) VALUES (?,
?)");
+ for (int i=0; i<n; ++i) {
+ final double x = mean[i];
+ pstmt.setDouble(1, x);
+ pstmt.setInt(2, i);
+ pstmt.executeUpdate();
+ }
+ }
+ finally {
+ if (stmt != null) { stmt.close(); stmt = null; }
+ if (pstmt != null) { pstmt.close(); pstmt = null; }
+ }
+
+ // Initialize sample-to-cluster assignment.
+ maximize();
+ }
+
+ /**
+ * (Re-)compute cluster centers while holding sample-to-cluster assignment fixed.
+ * @return mean square error of resulting clustering configuration
+ * @throws SQLException
+ */
+ double expect() throws SQLException
+ {
+ // working table names
+ final String clusterN = "cluster" + uniq;
+ final String sampleN = "sample" + uniq;
+ final String mm = "mm" + uniq;
+ final String vv = "vv" + uniq;
+ final String ee = "ee" + uniq;
+
+ double msr = Double.MAX_VALUE;
+ Statement stmt = null;
+ ResultSet rset = null;
+ try {
+ stmt = conn.createStatement();
+
+ stmt.executeUpdate("CREATE TABLE " + mm + "(bucket int PRIMARY KEY, mean double)");
+ stmt.executeUpdate("CREATE TABLE " + vv + "(bucket int PRIMARY KEY, var double)");
+ stmt.executeUpdate("CREATE TABLE " + ee + "(bucket int PRIMARY KEY, err double, size
int)");
+
+ stmt.executeUpdate("INSERT INTO " + mm + "(bucket, mean) \n" +
+ "SELECT bucket, avg(value) \n" +
+ " FROM " + sampleN + " \n" +
+ " GROUP BY bucket \n");
+
+ stmt.executeUpdate("INSERT INTO " + ee + "(bucket, err, size) \n" +
+ "SELECT S.bucket, sum((S.value - M.mean) * (S.value - M.mean)),
count(*) \n" +
+ " FROM " + sampleN + " S JOIN " + mm + " M ON S.bucket = M.bucket
\n" +
+ " GROUP BY S.bucket \n");
+
+ stmt.executeUpdate("INSERT INTO " + vv + "(bucket, var) \n" +
+ "SELECT bucket, \n" +
+ " CASE WHEN (size > 1) THEN (err / (size - 1)) ELSE 0.0
END \n" +
+ " FROM " + ee + " \n");
+
+ stmt.executeUpdate("DELETE FROM " + clusterN);
+
+ stmt.executeUpdate("INSERT INTO " + clusterN + "(mean, var, bucket) \n" +
+ "SELECT M.mean, V.var, V.bucket \n" +
+ " FROM " + mm + " M JOIN " + vv + " V ON M.bucket = V.bucket \n");
+
+ rset = stmt.executeQuery("SELECT (sum(err) / sum(size)) AS measure FROM " + ee);
+ while (rset.next()) { msr = rset.getDouble(1); }
+
+ stmt.executeUpdate("DROP TABLE " + mm);
+ stmt.executeUpdate("DROP TABLE " + vv);
+ stmt.executeUpdate("DROP TABLE " + ee);
+ }
+ finally {
+ if (rset != null) { rset.close(); rset = null; }
+ if (stmt != null) { stmt.close(); stmt = null; }
+ }
+ return msr;
+ }
+
+ /**
+ * (Re-)compute sample-to-cluster assignment while holding cluster centers fixed.
+ * @throws SQLException
+ */
+ void maximize() throws SQLException
+ {
+ // working table names
+ final String clusterN = "cluster" + uniq;
+ final String sampleN = "sample" + uniq;
+ final String gg = "gg" + uniq;
+ final String jj = "jj" + uniq;
+
+ Statement stmt = null;
+ try {
+ stmt = conn.createStatement();
+
+ stmt.executeUpdate("CREATE TABLE " + gg + "(id int PRIMARY KEY, diff double)");
+ stmt.executeUpdate("CREATE TABLE " + jj + "(value double, id int, diff double, bucket
int)");
+
+ stmt.executeUpdate("INSERT INTO " + gg + "(id, diff) \n" +
+ "SELECT S.id, min(abs(S.value - C.mean)) \n" +
+ " FROM " + sampleN + " S, " + clusterN + " C \n" +
+ " GROUP BY S.id \n");
+
+ stmt.executeUpdate("INSERT INTO " + jj + "(value, id, diff, bucket) \n" +
+ "SELECT S.value, S.id, abs(S.value - C.mean), C.bucket \n" +
+ " FROM " + sampleN + " S, " + clusterN + " C \n");
+
+ stmt.executeUpdate("DELETE FROM " + sampleN);
+
+ stmt.executeUpdate("INSERT INTO " + sampleN + "(value, id, bucket) \n" +
+ "SELECT J.value, J.id, min(J.bucket) \n" +
+ " FROM " + jj + " J \n" +
+ " JOIN " + gg + " G \n" +
+ " ON J.id = G.id \n" +
+ " AND J.diff = G.diff \n" +
+ " GROUP BY J.id, J.value \n");
+
+ stmt.executeUpdate("DROP TABLE " + gg);
+ stmt.executeUpdate("DROP TABLE " + jj);
+ }
+ finally {
+ if (stmt != null) { stmt.close(); }
+ }
+ }
+
+ /**
+ * Download the computed cluster configuration.
+ * @throws SQLException
+ */
+ void download() throws SQLException
+ {
+ // working table names
+ final String clusterN = "cluster" + uniq;
+ final String sampleN = "sample" + uniq;
+ final String ww = "ww" + uniq;
+ final String cc = "cc" + uniq;
+
+ Statement stmt = null;
+ ResultSet rset = null;
+ try {
+ stmt = conn.createStatement();
+
+ stmt.executeUpdate("CREATE TABLE " + ww + "(bucket int PRIMARY KEY, size int)");
+ stmt.executeUpdate("CREATE TABLE " + cc + "(weight double, mean double, var double,
bucket int PRIMARY KEY)");
+
+ stmt.executeUpdate("INSERT INTO " + ww + "(bucket, size) \n" +
+ "SELECT bucket, count(*) \n" +
+ " FROM " + sampleN + " \n" +
+ " GROUP BY bucket \n");
+
+ stmt.executeUpdate("INSERT INTO " + cc + "(weight, mean, var, bucket) \n" +
+ "SELECT (CAST(W.size AS double) / (SELECT sum(size) FROM " + ww
+ ")), C.mean, C.var, C.bucket \n" +
+ " FROM " + clusterN + " C JOIN " + ww + " W ON C.bucket = W.bucket
\n");
+
+ stmt.executeUpdate("DELETE FROM " + clusterN);
+
+ stmt.executeUpdate("INSERT INTO " + clusterN + "(weight, mean, var, bucket) \n" +
+ "SELECT weight, mean, var, bucket FROM " + cc + " \n");
+
+ stmt.executeUpdate("DROP TABLE " + ww);
+ stmt.executeUpdate("DROP TABLE " + cc);
+
+ rset = stmt.executeQuery("SELECT weight, mean, var FROM " + clusterN + " ORDER BY mean");
+ n = 0;
+ while (rset.next()) {
+ final double w = rset.getDouble(1);
+ final double m = rset.getDouble(2);
+ final double v = rset.getDouble(3);
+ weight[n] = w;
+ mean[n] = m;
+ var[n] = v;
+ ++n;
+ }
+ }
+ finally {
+ if (rset != null) { rset.close(); rset = null; }
+ if (stmt != null) { stmt.close(); stmt = null; }
+ }
+ }
+
+ /**
+ * Clean up working tables after use.
+ * @throws SQLException
+ */
+ void cleanup() throws SQLException
+ {
+ // working table names
+ final String clusterN = "cluster" + uniq;
+ final String sampleN = "sample" + uniq;
+
+ Statement stmt = null;
+ try {
+ stmt = conn.createStatement();
+ stmt.executeUpdate("DROP TABLE " + sampleN);
+ stmt.executeUpdate("DROP TABLE " + clusterN);
+ }
+ finally {
+ if (stmt != null) { stmt.close(); stmt = null; }
+ }
+ }
+
+ /** database connection */
+ Connection conn;
+
+ /** unique ID for generating working table names */
+ int uniq = 0;
+}
Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/ClusMogSQL.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/GenMog.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/GenMog.java?rev=759181&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/GenMog.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/GenMog.java
Fri Mar 27 15:45:59 2009
@@ -0,0 +1,91 @@
+/*
+ Derby - Class org.apache.derbyTesting.functionTests.tests.memorydb.GenMog
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ */
+
+package org.apache.derbyTesting.functionTests.tests.memorydb;
+
+import java.util.Random;
+import org.apache.derbyTesting.junit.BaseTestCase;
+
+/**
+ * Simple utility to generate a mixture-of-Gaussian configuration.
+ */
+public class GenMog
+{
+ /** default constructor */
+ public GenMog() { rng = new Random(System.currentTimeMillis()); }
+
+ /** constructor with specified RNG */
+ public GenMog(Random rng) { this.rng = rng; }
+
+ /** Generate a mixture-of-Gaussian configuration. */
+ public void generate()
+ {
+ // number of Gaussian components
+ n = 1 + rng.nextInt(max_n);
+
+ // weights associated with the Gaussian components
+ double sum = 0.0;
+ for (int i=0; i<n; ++i) {
+ double w;
+ do { w = rng.nextDouble(); } while (w <= 0.0);
+ weight[i] = w;
+ sum += w;
+ }
+ for (int i=0; i<n; ++i) { weight[i] /= sum; }
+
+ // (mean, var) parameters for the Gaussian components
+ double oldm = 0.0, olds = 0.0;
+ for (int i=0; i<n; ++i) {
+ final double s = min_s + rng.nextDouble() * (max_s - min_s);
+ final double m = oldm + 2.0 * olds + 2.0 * s;
+ mean[i] = m;
+ var[i] = s * s;
+ oldm = m;
+ olds = s;
+ }
+
+ // diagnostic messages
+ BaseTestCase.println("GENERATED: n = " + n);
+ for (int i=0; i<n; ++i) {
+ BaseTestCase.println("(w, m, v) = (" +
+ weight[i] + ", " + mean[i] + ", " + var[i] + ")");
+ }
+ }
+
+ /** random number generator in use */
+ public Random rng;
+
+ /** maximum number of Gaussian components */
+ public final static int max_n = 6;
+ /** minimum value for the standard deviation parameter */
+ public final static double min_s = 1.0;
+ /** maximum value for the standard deviation parameter */
+ public final static double max_s = 6.0;
+
+ /** actual number of Gaussian components */
+ public int n;
+ /** weights associated with the Gaussian components */
+ public final double weight[] = new double[max_n];
+ /** mean parameters for the Gaussian components */
+ public final double mean[] = new double[max_n];
+ /** variance parameters for the Gaussian components */
+ public final double var[] = new double[max_n];
+}
Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/GenMog.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/MogTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/MogTest.java?rev=759181&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/MogTest.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/MogTest.java
Fri Mar 27 15:45:59 2009
@@ -0,0 +1,194 @@
+/*
+ Derby - Class org.apache.derbyTesting.functionTests.tests.memorydb.MogTest
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ */
+
+package org.apache.derbyTesting.functionTests.tests.memorydb;
+
+import junit.framework.Test;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import org.apache.derbyTesting.junit.BaseJDBCTestCase;
+import org.apache.derbyTesting.junit.TestConfiguration;
+
+/**
+ * Test consistency among (GenMog), (SampMog), and (ClusMog).
+ */
+public class MogTest extends BaseJDBCTestCase {
+
+ public MogTest(String name) { super(name); }
+
+ public static Test suite() {
+ return TestConfiguration.defaultSuite(MogTest.class);
+ }
+
+ /** Dispose of objects after testing. */
+ protected void tearDown() throws Exception
+ {
+ super.tearDown();
+ }
+
+ /**
+ * Calculates by using the default directory/disk storage back end.
+ *
+ * @throws SQLException if the test fails
+ */
+ public void testClusMogOnDisk()
+ throws SQLException {
+ long start = System.currentTimeMillis();
+ doTestClusMog(getConnection());
+ println("duration-on-disk: " + (System.currentTimeMillis() - start));
+ }
+
+ /**
+ * Calculates by using the in-memory storage back end.
+ *
+ * @throws SQLException if the test fails
+ */
+ public void testClusMogInMemory()
+ throws SQLException {
+ long start = System.currentTimeMillis();
+ // Close the connection explicitly here, since the test framework won't
+ // do it for us when we create it manually.
+ Connection conn = obtainConnection();
+ try {
+ doTestClusMog(conn);
+ } finally {
+ try {
+ conn.rollback();
+ conn.close();
+ } catch (SQLException sqle) {
+ // Ignore exceptions during close.
+ }
+ }
+ println("duration-in-memory: " + (System.currentTimeMillis() - start));
+ }
+
+ /**
+ * Test consistency between (ClusMog) and (ClusMogSQL).
+ * @throws SQLException if something goes wrong
+ */
+ public void doTestClusMog(Connection conn)
+ throws SQLException {
+
+ // Initialize test objects.
+ long _seed = System.currentTimeMillis();
+ java.util.Random rng = new java.util.Random(_seed);
+ /** MOG generator being tested */
+ GenMog genMog = new GenMog(rng);
+ /** MOG sampler being tested */
+ SampMog sampMog = new SampMog(rng);
+ /** clustering object being tested */
+ ClusMog clusMog = new ClusMog();
+ /** clustering object being tested */
+ ClusMogSQL clusMogSql = new ClusMogSQL(conn);
+ clusMogSql.setUnique(rng.nextInt());
+
+ println(getName() + " using random seed: " + _seed);
+ final int max_ns = 10 * ClusMog.min_sample_size_per_cluster * GenMog.max_n;
+ final double sample[] = new double[max_ns];
+ final double center[] = new double[GenMog.max_n];
+ final int niter = 1;
+ for (int i=niter; i>0; --i) {
+ // Generate a MOG configuration.
+ genMog.generate();
+ // Compute a sample size.
+ final int min_ns = ClusMog.min_sample_size_per_cluster * genMog.n;
+ final int ns = min_ns + rng.nextInt(max_ns - min_ns);
+ println("ns = " + ns);
+ // Generate samples from the MOG distribution.
+ sampMog.set(genMog.n, genMog.weight, genMog.mean, genMog.var);
+ sampMog.generate(ns, sample);
+
+ // Produce an initial cluster center configuration.
+ ClusMog.uniform(genMog.n, center, ns, sample);
+ // Cluster the samples to recover the MOG configuration.
+ clusMog.cluster(genMog.n, center, ns, sample);
+ // Cluster the samples again, using SQL.
+ clusMogSql.clusterSQL(genMog.n, center, ns, sample);
+ // Compare the computed MOG configurations.
+ assertEquals(clusMog.n, clusMogSql.n);
+ compare(clusMog.n, clusMog.weight, clusMogSql.weight);
+ compare(clusMog.n, clusMog.mean, clusMogSql.mean);
+ compare(clusMog.n, clusMog.var, clusMogSql.var);
+
+ // Produce another initial cluster center configuration.
+ ClusMog.random(genMog.n, center, ns, sample, rng);
+ // Cluster the samples to recover the MOG configuration.
+ clusMog.cluster(genMog.n, center, ns, sample);
+ // Cluster the samples again, using SQL.
+ clusMogSql.clusterSQL(genMog.n, center, ns, sample);
+ // Compare the computed MOG configurations.
+ assertEquals(clusMog.n, clusMogSql.n);
+ compare(clusMog.n, clusMog.weight, clusMogSql.weight);
+ compare(clusMog.n, clusMog.mean, clusMogSql.mean);
+ compare(clusMog.n, clusMog.var, clusMogSql.var);
+ }
+ }
+
+ /** Compare two floating-point arrays, with tolerance. */
+ private void compare(int n, double ones[], double oths[])
+ {
+ final double thresh = 1.0e-6;
+ for (int i=0; i<n; ++i) {
+ final double one = ones[i];
+ final double oth = oths[i];
+ final double dif = Math.abs(one - oth);
+ final double err = dif / (1.0 + Math.abs(one));
+ assertTrue(err < thresh);
+ }
+ }
+
+ /**
+ * Obtains a connection to an in-memory database.
+ *
+ * @return A connection to an in-memory database.
+ * @throws SQLException if obtaining the connection fails
+ */
+ private Connection obtainConnection()
+ throws SQLException {
+ try {
+ if (usingDerbyNetClient()) {
+ Class.forName("org.apache.derby.jdbc.ClientDriver");
+ } else {
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+ }
+ } catch (Exception e) {
+ SQLException sqle = new SQLException(e.getMessage());
+ sqle.initCause(e);
+ throw sqle;
+ }
+ StringBuffer sb = new StringBuffer("jdbc:derby:");
+ if (usingEmbedded()) {
+ sb.append("memory:");
+ } else {
+ // This is a hack. Change this when proper support for the in-memory
+ // back end has been implemented.
+ sb.append("//");
+ sb.append(TestConfiguration.getCurrent().getHostName());
+ sb.append(':');
+ sb.append(TestConfiguration.getCurrent().getPort());
+ sb.append('/');
+ sb.append("memory:");
+ }
+ sb.append("MogTestDb;create=true");
+ return DriverManager.getConnection(sb.toString());
+ }
+}
Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/MogTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/SampMog.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/SampMog.java?rev=759181&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/SampMog.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/SampMog.java
Fri Mar 27 15:45:59 2009
@@ -0,0 +1,168 @@
+/*
+ Derby - Class org.apache.derbyTesting.functionTests.tests.memorydb.SampMog
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ */
+
+package org.apache.derbyTesting.functionTests.tests.memorydb;
+
+import java.util.Arrays;
+import java.util.Random;
+
+/**
+ * Simple utility to generate samples from a mixture-of-Gaussian distribution.
+ */
+public class SampMog
+{
+ /** default constructor */
+ public SampMog() { rng = new Random(System.currentTimeMillis()); }
+
+ /** constructor with specified RNG */
+ public SampMog(Random rng) { this.rng = rng; }
+
+ /**
+ * Generate samples from the specified mixture-of-Gaussian distribution.
+ * @param ns number of samples to generate
+ * @param sample output array of generated samples
+ */
+ public void generate(int ns, double sample[])
+ {
+ // First, generate standard normal samples, drawn from N(0, 1).
+ for (int i=0; i<ns; ++i) {
+ final double r = rng.nextGaussian();
+ sample[i] = r;
+ }
+ // Then, transform the samples to conform to the Gaussian components
+ // according to their weights.
+ for (int i=0; i<ns; ++i) {
+ // Pick a Gaussian component, represented by (idx).
+ final double w = rng.nextDouble();
+ int idx = Arrays.binarySearch(cumulative, w);
+ if (idx < 0) { idx = -(idx + 1); }
+ if (idx >= n) { idx = n-1; }
+ final double m = mean[idx];
+ final double s = stdv[idx];
+ // Transform the sample to conform to the Gaussian component.
+ final double r = sample[i];
+ final double x = m + s * r;
+ sample[i] = x;
+ }
+ }
+
+ /** Get the maximum number of Gaussian components. */
+ public static int getMaxNumber() { return max_n; }
+
+ /** Get the number of Gaussian components. */
+ public int getNumber() { return n; }
+
+ /** Specify the mixture-of-Gaussian configuration. */
+ public void set(int n, double wts[], double mm[], double vv[])
+ {
+ setNumber(n);
+ setWeights(wts);
+ setMeans(mm);
+ setVars(vv);
+ }
+
+ /** Set the number of Gaussian components. */
+ public void setNumber(int n)
+ {
+ if (1 <= n && n <= max_n) {
+ this.n = n;
+ }
+ else {
+ final String msg =
+ "Number of Gaussian components (" + n + ") not in [1, " + max_n + "].";
+ throw new IllegalArgumentException(msg);
+ }
+ }
+
+ /** Specify the weights for the Gaussian components. */
+ public void setWeights(double wts[])
+ {
+ // Copy weights to internal state array.
+ System.arraycopy(wts, 0, weight, 0, n);
+
+ // Normalize the weights to sum to 1.
+ IllegalArgumentException ex = null;
+ double sum = 0.0;
+ for (int i=0; i<n; ++i) {
+ final double wt = weight[i];
+ if (wt > 0.0) {
+ sum += wt;
+ }
+ else {
+ if (ex == null) {
+ final String msg = "Invalid weight (" + wt + ").";
+ ex = new IllegalArgumentException(msg);
+ }
+ }
+ }
+ if (sum > 0.0) {
+ for (int i=0; i<n; ++i) { weight[i] /= sum; }
+ }
+ else {
+ if (ex == null) {
+ final String msg = "Invalid total weight (" + sum + ").";
+ ex = new IllegalArgumentException(msg);
+ }
+ }
+ if (ex != null) { throw ex; }
+
+ // Compute cumulative weights.
+ cumulative[0] = weight[0];
+ for (int i=1; i<n; ++i) { cumulative[i] = Math.min(cumulative[i-1] + weight[i], 1.0);
}
+ for (int i=n; i<max_n; ++i) { cumulative[i] = 1.0; }
+ }
+
+ /** Specify the mean parameters for the Gaussian components. */
+ public void setMeans(double mm[])
+ {
+ System.arraycopy(mm, 0, mean, 0, n);
+ }
+
+ /** Specify the variance parameters for the Gaussian components. */
+ public void setVars(double vv[])
+ {
+ for (int i=0; i<n; ++i) {
+ final double v = Math.abs(vv[i]);
+ var[i] = v;
+ stdv[i] = Math.sqrt(v);
+ }
+ }
+
+ /** random number generator in use */
+ public Random rng;
+
+ /** maximum number of Gaussian components */
+ public final static int max_n = 6;
+
+ /** actual number of Gaussian components */
+ int n = 3;
+ /** weights associated with the Gaussian components */
+ final double weight[] = new double[max_n];
+ /** mean parameters for the Gaussian components */
+ final double mean[] = new double[max_n];
+ /** variance parameters for the Gaussian components */
+ final double var[] = new double[max_n];
+
+ /** cumulative weights, for sample generation */
+ final double cumulative[] = new double[max_n];
+ /** standard deviation parameters for the Gaussian components */
+ final double stdv[] = new double[max_n];
+}
Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/SampMog.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/_Suite.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/_Suite.java?rev=759181&r1=759180&r2=759181&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/_Suite.java
(original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memorydb/_Suite.java
Fri Mar 27 15:45:59 2009
@@ -22,6 +22,7 @@
package org.apache.derbyTesting.functionTests.tests.memorydb;
import org.apache.derbyTesting.junit.BaseTestCase;
+import org.apache.derbyTesting.junit.JDBC;
import junit.framework.Test;
import junit.framework.TestSuite;
@@ -42,8 +43,11 @@
public static Test suite() {
TestSuite suite = new TestSuite("In-memory db test suite");
- suite.addTest(BasicInMemoryDbTest.suite());
-
+ // Tests are not yet compatible with JSR169 / JavaME
+ if (!JDBC.vmSupportsJSR169()) {
+ suite.addTest(BasicInMemoryDbTest.suite());
+ suite.addTest(MogTest.suite());
+ }
return suite;
}
}
|