From olio-commits-return-5-apmail-incubator-olio-commits-archive=incubator.apache.org@incubator.apache.org Mon Oct 20 17:40:22 2008 Return-Path: Delivered-To: apmail-incubator-olio-commits-archive@locus.apache.org Received: (qmail 19623 invoked from network); 20 Oct 2008 17:40:22 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 20 Oct 2008 17:40:22 -0000 Received: (qmail 51658 invoked by uid 500); 20 Oct 2008 17:40:25 -0000 Delivered-To: apmail-incubator-olio-commits-archive@incubator.apache.org Received: (qmail 51646 invoked by uid 500); 20 Oct 2008 17:40:25 -0000 Mailing-List: contact olio-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: olio-dev@incubator.apache.org Delivered-To: mailing list olio-commits@incubator.apache.org Received: (qmail 51635 invoked by uid 99); 20 Oct 2008 17:40:25 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 20 Oct 2008 10:40:25 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 20 Oct 2008 17:39:14 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 6A1832388A1E; Mon, 20 Oct 2008 10:39:22 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r706345 [4/5] - in /incubator/olio/workload/rails: ./ trunk/ trunk/bin/ trunk/config/ trunk/config/security/ trunk/deploy/ trunk/lib/ trunk/mysql-connector-java-5.0.6/ trunk/mysql-connector-java-5.0.6/debug/ trunk/mysql-connector-java-5.0.6... Date: Mon, 20 Oct 2008 17:39:20 -0000 To: olio-commits@incubator.apache.org From: wsobel@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20081020173922.6A1832388A1E@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Added: incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/Tag.java URL: http://svn.apache.org/viewvc/incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/Tag.java?rev=706345&view=auto ============================================================================== --- incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/Tag.java (added) +++ incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/Tag.java Mon Oct 20 10:39:16 2008 @@ -0,0 +1,72 @@ +package com.sun.web20.loader; + +import com.sun.web20.util.UserName; +import com.sun.web20.loader.framework.Loadable; +import com.sun.web20.loader.framework.Loader; +import com.sun.web20.loader.framework.ThreadConnection; + +import java.util.logging.Logger; +import java.util.logging.Level; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +/** + * The tag loader. + */ +public class Tag extends Loadable { + + // Note that the tag id in the database is autoincrement and may + // not coincide with this tag id/name when using multi-thread loading. + private static final String STATEMENT = "insert into tags " + + "(name, id) values (?, ?)"; + + static Logger logger = Logger.getLogger(Tag.class.getName()); + + int id; + String tag; + + public Tag(int id) { + this.id = ++id; + } + + public String getClearStatement() { + return "truncate table tags"; + } + + public void prepare() { + tag = UserName.getUserName(id); + } + + + public void load() { + ThreadConnection c = ThreadConnection.getInstance(); + try { + PreparedStatement s = c.prepareStatement(STATEMENT); + s.setString(1, tag); + s.setInt(2, id); + c.addBatch(); + } catch (SQLException e) { + logger.log(Level.SEVERE, e.getMessage(), e); + Loader.increaseErrorCount(); + } + } + + /** + * For tags, we won't know the refcount till all the data is loaded. + * So we update the table at postload. + */ + public void postLoad() { +// ThreadConnection c = ThreadConnection.getInstance(); +// try { +// c.prepareStatement("update SOCIALEVENTTAG set refcount = " + +// "(select count(*) from SOCIALEVENTTAG_SOCIALEVENT " + +// "where socialeventtagid = " + +// "SOCIALEVENTTAG.socialeventtagid)"); +// c.executeUpdate(); +// } catch (SQLException e) { +// logger.log(Level.SEVERE, e.getMessage(), e); +// } + + + } +} Added: incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/Loadable.java URL: http://svn.apache.org/viewvc/incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/Loadable.java?rev=706345&view=auto ============================================================================== --- incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/Loadable.java (added) +++ incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/Loadable.java Mon Oct 20 10:39:16 2008 @@ -0,0 +1,17 @@ +package com.sun.web20.loader.framework; + +public abstract class Loadable { + + protected Loader loader = Loader.getInstance(getClass().getName()); + + public abstract String getClearStatement(); + + public abstract void prepare(); + + public abstract void load(); + + public void postLoad() { + // Empty. We do not make it abstract. + // A majority of LoadObjects do not need this. + } +} Added: incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/Loader.java URL: http://svn.apache.org/viewvc/incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/Loader.java?rev=706345&view=auto ============================================================================== --- incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/Loader.java (added) +++ incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/Loader.java Mon Oct 20 10:39:16 2008 @@ -0,0 +1,235 @@ +package com.sun.web20.loader.framework; + +import java.sql.SQLException; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * The loader, one instance per Loadable type loaded, is responsible + * for loading the data into the database in the most efficient manner. + * We use queues and thread pools to ensure multiple threads are loading + * concurrently in large batches. + * + * @author Akara Sucharitakul + */ +public class Loader { + + /** The batch size of a single batch. */ + public static final int BATCHSIZE = 1000; + + /** The number of errors before exiting. */ + public static final int ERROR_THRESHOLD = 100; + + public static final int LOAD_THREADS = 5; + + private static Logger logger = + Logger.getLogger(Loader.class.getName()); + + private static AtomicInteger errorCount = new AtomicInteger(); + + private static ConcurrentHashMap typeMap = + new ConcurrentHashMap(); + + // This is a single processing pool for processing data preps. + private static ExecutorService processor = + Executors.newCachedThreadPool(); + + private String name; + AtomicInteger loadCount; + + // A Loadable type database loading pool. + ExecutorService pool; + ConcurrentLinkedQueue queue; + + /** + * Obtains the instance of the loader for a given loadable type. + * @param name The loadable type name + * @return The loader for this type name, or a new loader if none exists + */ + static Loader getInstance(String name) { + // We may need to change this to a configurable thread pool size + // on a per-type basis. This is the only place to change. + + Loader loader = new Loader(); + Loader oldEntry = typeMap.putIfAbsent(name, loader); + + if (oldEntry != null) + loader = oldEntry; + + loader.validate(name); + return loader; + } + + private synchronized void validate(String name) { + if (this.name == null) + this.name = name; + if (loadCount == null) + loadCount = new AtomicInteger(0); + if (queue == null) + queue = new ConcurrentLinkedQueue(); + + // We may need to change this to a configurable thread pool size + // on a per-type basis. This is the only place to change. + if (pool == null) + pool = Executors.newFixedThreadPool(LOAD_THREADS); + // pool = Executors.newCachedThreadPool(); + } + + /** + * Sets the URL for the connection to the database. + * @param url The connection URL + */ + public static void setConnectionURL(String url) { + ThreadConnection.connectionURL = url; + } + + public static void setJDBCDriverClassName(String driver) + throws ClassNotFoundException, IllegalAccessException, + InstantiationException { + // Just load the DB driver class. + Class.forName(driver).newInstance(); + } + + /** + * Uses the loadable to clear the database through the loadable's + * clear statement. + * @param l The loadable to use + */ + public static void clear(final Loadable l) { + Future f = l.loader.pool.submit(new Runnable() { + public void run() { + ThreadConnection c = ThreadConnection.getInstance(); + try { + c.prepareStatement(l.getClearStatement()); + c.executeUpdate(); + } catch (SQLException e) { + logger.log(Level.SEVERE, l.loader.name + ": " + + e.getMessage(), e); + increaseErrorCount(); + } + } + }); + while (!f.isDone() || f.isCancelled()) { + try { + Thread.sleep(200); + } catch (InterruptedException e) { + logger.log(Level.WARNING, l.loader.name + ": Interrupted while " + + "waiting to clear table.", e); + } + } + } + + /** + * Loads the loadable into the database. Note that the loading is done + * asynchronously and is divided into two phases: 1) The preparation + * phase where all field values are generated and 2) Loading phase. These + * may be performed by different threads. The waitProcessing method + * will gracefully shut down the processing infrastructure and wait until + * all preparation is done. Shutdown will wait until all data loading + * is done. + * @param l + */ + public static void load(final Loadable l) { + processor.execute(new Runnable() { + public void run() { + try { + l.prepare(); + l.loader.add(l); + } catch (Exception e) { + logger.log(Level.WARNING, e.getMessage(), e); + Loader.increaseErrorCount(); + } + } + }); + } + + /** + * Execute the post loads provided by the loadable. + * @param l The loadable. + */ + public static void postLoad(final Loadable l) { + l.loader.pool.submit(new Runnable() { + public void run() { + try { + l.postLoad(); + } catch (Exception e) { + logger.log(Level.WARNING, l.loader.name + ": " + + e.getMessage(), e); + increaseErrorCount(); + } + } + }); + } + + + private void add(Loadable l) { + queue.add(l); + int c = loadCount.incrementAndGet(); + if (c % BATCHSIZE == 0) + flush(c); + } + + private void flush(final int batchCount) { + pool.submit(new Runnable() { + public void run() { + ThreadConnection c = ThreadConnection.getInstance(); + c.processBatch(name, batchCount, queue); + } + }); + } + + /** + * Terminates the preparation infrastructure and waits until all data + * preparation is done. + */ + public static void waitProcessing() { + // We ensure the process pool is cleared, first. + if (processor != null) { + processor.shutdown(); + boolean terminated = false; + while (!terminated) + try { + terminated = processor.awaitTermination(1, TimeUnit.HOURS); + } catch (InterruptedException e) { + } + processor = null; + } + } + + /** + * Terminates the preparation infrastructure (if still alive) and + * then the loading infrastructure. Will return only after all the + * loadables in the queue are loaded. + */ + public static void shutdown() { + waitProcessing(); + for (Loader entry : typeMap.values()) + entry.flush(0); + for (Loader entry : typeMap.values()) + entry.pool.shutdown(); + for (Loader entry : typeMap.values()) { + while (!entry.pool.isTerminated()) + try { + entry.pool.awaitTermination(1, TimeUnit.HOURS); + } catch (InterruptedException e) { + } + } + typeMap.clear(); + ThreadConnection.closeConnections(); + } + + /** + * Increments the global error count. If the count is beyond the threshold, + * the loader will terminate. + */ + public static void increaseErrorCount() { + if (errorCount.incrementAndGet() > ERROR_THRESHOLD) + logger.severe("Error count exceeded threshold of " + + ERROR_THRESHOLD + "! Exiting."); + System.err.println("Error count exceeded threshold of " + + ERROR_THRESHOLD + "! Exiting."); + System.exit(2); + } +} Added: incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/ThreadConnection.java URL: http://svn.apache.org/viewvc/incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/ThreadConnection.java?rev=706345&view=auto ============================================================================== --- incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/ThreadConnection.java (added) +++ incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/ThreadConnection.java Mon Oct 20 10:39:16 2008 @@ -0,0 +1,242 @@ +package com.sun.web20.loader.framework; + +import java.sql.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Queue; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Wraps a java.sql.connection on a per-thread basis. + */ +public class ThreadConnection { + + private static Logger logger = + Logger.getLogger(ThreadConnection.class.getName()); + + private static ThreadLocal resource = + new ThreadLocal() { + public ThreadConnection initialValue() { + return new ThreadConnection(); + } + }; + + private static boolean COMMIT_TX = Boolean.parseBoolean( + System.getProperty("commit.tx", "true")); + private static final List CONNECTIONLIST = + Collections.synchronizedList(new ArrayList()); + + public static String connectionURL; + + private Connection conn; + private String statementText; + private PreparedStatement statement; + private int currentBatch; + private boolean closed = false; + + /** + * The batch buffer buffers the loadables to be added in a batch. + * These have to be per-thread and the only reason they are + * maintained by the ThreadConnection. Otherwise we need to keep + * allocating and collecting. And since we already have the threadlocal, + * there is no more overhead getting to it. + */ + Loadable[] batchBuffer; + + private ThreadConnection() { + CONNECTIONLIST.add(this); + } + + public static ThreadConnection getInstance() { + return resource.get(); + } + + boolean ensureConnection() { + if (closed) { + logger.severe("Connection used after closure!"); + Loader.increaseErrorCount(); + return false; + } + + try { + if (conn == null || conn.isClosed()) { + conn = DriverManager.getConnection(connectionURL); + //conn.setAutoCommit(false); + statement = null; + statementText = null; + } + } catch (SQLException e) { + logger.log(Level.SEVERE, "Error connecting to DB", e); + Loader.increaseErrorCount(); + return false; + } + return true; + } + + boolean resetConnection() { + if (closed) { + logger.severe("Connection used after closure!"); + Loader.increaseErrorCount(); + return false; + } + + try { + conn = DriverManager.getConnection(connectionURL); + statement = null; + statementText = null; + } catch (SQLException e) { + logger.log(Level.SEVERE, "Error connecting to DB", e); + Loader.increaseErrorCount(); + return false; + } + return true; + } + + public PreparedStatement prepareStatement(String statementText) + throws SQLException { + if (conn == null) + ensureConnection(); + if (!statementText.equals(this.statementText)) { + this.statementText = statementText; + statement = conn.prepareStatement(statementText); + } + return statement; + } + + public void addBatch() throws SQLException { + statement.addBatch(); + ++currentBatch; + } + + public void executeUpdate() throws SQLException { + if (ensureConnection()) { + statement.executeUpdate(); + if (COMMIT_TX) + conn.commit(); + } + } + + void processBatch(String name, int batchCount, Queue queue) { + // First we need to save the load objects from the queue + // so we do not loose them in case we need to retry. + if (batchBuffer == null) { + batchBuffer = new Loadable[Loader.BATCHSIZE]; + } + int count = 0; + for (; count < Loader.BATCHSIZE; count++) { + Loadable l = queue.poll(); + if (l == null) + break; + batchBuffer[count] = l; + } + + if (count == 0) // Nothing to load. + return; + + // Then we load our objects into the DB, retrying the whole + // saved ones in case we run into a closed connection. + if (!ensureConnection()) + return; + + String batchName; + if (batchCount > 0) + batchName = "object batch " + (batchCount - count + 1) + " - " + + batchCount + '.'; + else + batchName = "final " + count + " object batch."; + + int flushed = 0; + for (int retry = 0; retry < 10; retry++) { + try { + for (int i = flushed; i < count; i++) { + batchBuffer[i].load(); + + // Each Loadable object may load more than 1 record. + // So we need to check for the number of records + // in the batch. If it is more than batchsize, we + // need to flush the records, too. + if (currentBatch >= Loader.BATCHSIZE) { + flush(); + flushed += currentBatch; + currentBatch = 0; + logger.finer(name + ": Flushed " + flushed + + " records in " + batchName); + } + } + if (currentBatch > 0) { + flush(); + flushed += currentBatch; + currentBatch = 0; + logger.finer(name + ": Flushed final " + flushed + + " records in " + batchName); + } + logger.fine(name + ": Loaded " + batchName); + break; // We won't retry if everything is OK. + } catch (BatchUpdateException e) { + if (retry < 10) { + resetConnection(); + logger.log(Level.WARNING, name + + ": Retry loading.", e); + } else { + int[] stats = e.getUpdateCounts(); + int successes = 0; + for (int stat : stats) { + if (stat != Statement.EXECUTE_FAILED) + ++successes; + } + if (successes == 0) { + logger.log(Level.WARNING, name + + ": Failed to update.", e); + Loader.increaseErrorCount(); + } + } + } catch (SQLException e) { + if (retry == 0) { + resetConnection(); + logger.log(Level.WARNING, name + ": Retry loading.", + e); + } else { + logger.log(Level.WARNING, e.getMessage(), e); + Loader.increaseErrorCount(); + } + } + } + + // Once we're done with this buffer, don't hold on to the objects. + // Let them get GC'd so we don't bloat memory. Minimal CPU cost + // for such tight loop and setting all entries to null. + for (int i = 0; i < batchBuffer.length; i++) + batchBuffer[i] = null; + } + + void flush() throws SQLException { + statement.executeBatch(); + if (COMMIT_TX) + conn.commit(); + } + + + void close() throws SQLException { + closed = true; + if (statement != null) + statement.close(); + if (conn != null) + conn.close(); + } + + static void closeConnections() { + System.out.println("Closing connection"); + synchronized (CONNECTIONLIST) { + for (ThreadConnection c : CONNECTIONLIST) + try { + c.close(); + } catch (SQLException e) { + logger.log(Level.SEVERE, e.getMessage(), e); + Loader.increaseErrorCount(); + } + CONNECTIONLIST.clear(); + } + } +} Added: incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/ThreadResource.java URL: http://svn.apache.org/viewvc/incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/ThreadResource.java?rev=706345&view=auto ============================================================================== --- incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/ThreadResource.java (added) +++ incubator/olio/workload/rails/trunk/src/com/sun/web20/loader/framework/ThreadResource.java Mon Oct 20 10:39:16 2008 @@ -0,0 +1,52 @@ +package com.sun.web20.loader.framework; + +import com.sun.faban.driver.util.Random; + +import java.util.logging.Logger; +import java.text.SimpleDateFormat; +import java.text.DateFormat; + +public class ThreadResource { + + Logger logger = Logger.getLogger(ThreadResource.class.getName()); + + private static ThreadLocal resource = + new ThreadLocal() { + public ThreadResource initialValue() { + return new ThreadResource(); + } + }; + + Random random; + StringBuilder buffer; + SimpleDateFormat dateFormat; + + private ThreadResource() { + buffer = new StringBuilder(256); + random = new Random(); + } + + public StringBuilder getBuffer() { + buffer.setLength(0); // Make sure we clear it + return buffer; + } + + public Random getRandom() { + return random; + } + + /** + * DateFormat is not thread safe. We need to include it into the + * ThreadResource. + * @return The thread instance of DateFormat. + */ + public DateFormat getDateFormat() { + if (dateFormat == null) + dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + return dateFormat; + } + + public static ThreadResource getInstance() { + return resource.get(); + } +} Added: incubator/olio/workload/rails/trunk/src/com/sun/web20/util/GrowthTest.java URL: http://svn.apache.org/viewvc/incubator/olio/workload/rails/trunk/src/com/sun/web20/util/GrowthTest.java?rev=706345&view=auto ============================================================================== --- incubator/olio/workload/rails/trunk/src/com/sun/web20/util/GrowthTest.java (added) +++ incubator/olio/workload/rails/trunk/src/com/sun/web20/util/GrowthTest.java Mon Oct 20 10:39:16 2008 @@ -0,0 +1,52 @@ +package com.sun.web20.util; + +/** + * Code to test the growth function and match it to existing research. + */ +public class GrowthTest { + + // We use on average of 3.5 tags per event. Random 1..6 tags. + // Once we know the tag count, we have to select tags. + + /* + + http://tagsonomy.com/index.php/dynamic-growth-of-tag-clouds/ + + As of this writing, a little over 700 users have tagged it, with 450+ + unique tags, roughly two-thirds of which tags were (of course) used by + one and only one user. + + It took only 10 users (not quite 1.5% of the current total) before the + top 3 tags were tagging ontology folksonomy, conveying much the same + sense, with only the use of tagging instead of tags making this + different from the current set of 3. + */ + + + public static double cumuLogistic( + double x, double mean, double scale) { + return 0.5d + Math.tanh((x - mean) / (2 * scale)) / 2d; + } + + public static double cumuHalfLogistic(double x, double scale) { + return (1d - Math.pow(Math.E, -x/scale)) / (1d + Math.pow(Math.E, -x/scale)); + } + + public static double sigmoid(double x, double mean, double scale) { + return 1d / (1d + Math.pow(Math.E, -((x / scale) - mean))); + } + + public static void main(String[] args) { + + int limit = 5000; + int mean = 5000; + int scale = 500; + for (int x = 0; x < 10000; x += 100) { + int y = (int) Math.round(limit * cumuLogistic(x, 5000, 1000)); + int y2 = (int) Math.round(limit * cumuHalfLogistic(x, 10000)); // Done + int y3 = (int) Math.round(limit * sigmoid(x, 6, 1000)); + System.out.println("-> " + x + ',' + y + ',' + y2 + ',' + y3); + } + } + +} Added: incubator/olio/workload/rails/trunk/src/com/sun/web20/util/MemCacheUtility.java URL: http://svn.apache.org/viewvc/incubator/olio/workload/rails/trunk/src/com/sun/web20/util/MemCacheUtility.java?rev=706345&view=auto ============================================================================== --- incubator/olio/workload/rails/trunk/src/com/sun/web20/util/MemCacheUtility.java (added) +++ incubator/olio/workload/rails/trunk/src/com/sun/web20/util/MemCacheUtility.java Mon Oct 20 10:39:16 2008 @@ -0,0 +1,308 @@ +/* +* MemCacheUtility.java +* +* Created on August 20, 2007, 10:23 PM +* +* To change this template, choose Tools | Template Manager +* and open the template in the editor. +*/ + +package com.sun.web20.util; + +import com.danga.MemCached.MemCachedClient; +import com.danga.MemCached.SockIOPool; +import com.sun.faban.common.NameValuePair; +import com.sun.faban.common.TextTable; + +import java.util.*; +import java.util.logging.Logger; + + + +/** + * + * @author Kim LiChong + */ +public class MemCacheUtility { + + private static MemCachedClient cache = null; + private String[] serverList = null; + static Logger logger = Logger.getLogger( + MemCacheUtility.class.getName()); + private long baseTime = Long.MIN_VALUE; + + + /** This constructor creates a new instance of MemCacheUtility + A memcache client is created with a pool of servers. + * + * @param servers ArrayList of NameValuePair of servers:port numbers. + */ + public MemCacheUtility(ArrayList> servers) { + if (cache == null) { + // the env memcachedInstances is in the + // form host1:port1, host2:port2, etc. + // in an ArrayList + //String servers = locator.getString("memcachedInstances"); + + + serverList = new String[servers.size()]; + + serverList = convertNameValueToStringArray(servers); + + //logger.info("size of the array is " + serverList.length); + //String[] serverList = servers.split(",?[ *]"); + SockIOPool pool = SockIOPool.getInstance("livePool"); + pool.setServers(serverList); + pool.initialize(); + + cache = new MemCachedClient(); + cache.setPoolName("livePool"); + } + } + + /** This constructor creates a new instance of MemCacheUtility + A memcache client is created with a pool of servers. + * + * @param servers String [] servers:port. + */ + public MemCacheUtility(String[] servers) { + if (cache == null) { + SockIOPool pool = SockIOPool.getInstance("livePool"); + pool.setServers(servers); + pool.initialize(); + + cache = new MemCachedClient(); + cache.setPoolName("livePool"); + } + } + + /* + * This method is a convenience method to convert ArrayList to + * a String array of server:port server2:port. + * @param servers ArrayList> + * @return String [] + */ + + public static String[] convertNameValueToStringArray ( + ArrayList> servers) { + String [] serverArr = new String[servers.size()]; + int index = 0; + for (NameValuePair serverEntry : servers) { + serverArr[index++] = serverEntry.name + ":" + serverEntry.value; + } + return serverArr; + } + + /** Retrieves memcache stats for each instance of MemCacheUtility. + * A TextTable will be produced for each server used to create an + * instance of MemCacheUtility. Used to persist stats file for each server. + * Returning Map whose key is the servername, the value is a TextTable of statistics + * @return Map + */ + + public Map getStats() { + + + Map memcacheStats = cache.stats(); + //cache.stats() will return a Map whose key is the name of the memcache server + //and whose value is a Map with the memcache statistics + + + //logger.info("Map size returning is " + memcacheStats.size()); + + //produce a TextTable for each server listed + Map returnMap = new HashMap(); + + TextTable outputTextTable = null; + + Set statEntries = memcacheStats.entrySet(); + + //set counter to allow to set number of columns to output + for (Map.Entry statEntry : statEntries) { + String key = (String) statEntry.getKey(); + Map statsMap = (Map) statEntry.getValue(); + //is this case, it is a Map with the statistics + //get size so we know how big to make TextTable + outputTextTable = new TextTable(statsMap.size(), 2); + //set Header + outputTextTable.setHeader(0, "Parameter"); + outputTextTable.setHeader(1, "Value"); + //outputTextTable.setHeader(2, "for " + key); + //get this value's iterator + Set statsMapEntries = statsMap.entrySet(); + int counter=0; + for (Map.Entry statsMapEntry : statsMapEntries) { + outputTextTable.setField(counter, 0, + (CharSequence) statsMapEntry.getKey()); + outputTextTable.setField(counter++, 1, + (CharSequence) statsMapEntry.getValue()); + } + //add each TextTable for each server listed to return Map. + returnMap.put(key, outputTextTable); + + } + return returnMap; + } + + /* This method is used for dynamic memcache stats gathering. + * The TextTable will contain all memcache server instances in columns + * and the server parameters in rows + * @return TextTable + *@see com.sun.faban.common.TextTable + */ + + public TextTable getTemporaryStats() { + Long time = System.currentTimeMillis(); + int elapsed = 0; + if (baseTime == Long.MIN_VALUE) + baseTime = time; + else + elapsed = (int) (time - baseTime); + + String elapsedSecs = String.format("%.3f", elapsed/1000d); + + Map memcacheStats = cache.stats(); + //cache.stats() will return a Map whose key is the name of the memcache server + //and whose value is a Map with the memcache statistics + TextTable outputTextTable = null; + Set serverEntries = memcacheStats.entrySet(); + + //set counter to allow to set number of columns to output + int counter = 0; + int columnIndex = 0; + + //reset the iterator + for (Map.Entry serverEntry : serverEntries) { + String key = (String) serverEntry.getKey(); + Map statsMap = (Map) serverEntry.getValue(); + if (outputTextTable == null) { + // One extra row for elapsed time, one extra header column. + outputTextTable = new TextTable(statsMap.size(), + serverEntries.size() + 2); + } + //is this case, it is a Map with the statistics + //get size so we know how big to make TextTable + // the number of rows is the number of stats + // the number of columns is how many server instances there are + //set Header + outputTextTable.setHeader(0, "Elapsed (sec)"); + outputTextTable.setHeader(1, "Parameter"); + outputTextTable.setHeader(columnIndex + 2, key); + + //get this value's iterator + Set statsMapEntries = statsMap.entrySet(); + counter=0; //reset counter + + // Populate the rest of the table. + for (Map.Entry statsMapEntry : statsMapEntries) { + outputTextTable.setField(counter, 0, elapsedSecs); + outputTextTable.setField(counter, 1, + (CharSequence) statsMapEntry.getKey()); + outputTextTable.setField(counter++, columnIndex + 2, + (CharSequence) statsMapEntry.getValue()); + } + ++columnIndex; + } + return outputTextTable; + } + + /* + This main method is used to gather dynamic statistics on memcache server instances. + * It expects at least 4 arguments: + * + * host:server host:server (additional server instances can be designated as host1:port1 host1:port2 OR host2:port etc. + * -s start time: the ramp up time, in seconds. (status collection does not take place during the ramp up) + * -e end time: the steady state, in seconds. (time to do the statistics data collection) + * -i interval time: the snapshot period to collect the stats, in seconds. + * + * Usage: java com.sun.web20.MemCacheUtility server:port [server2:port server3:port] -s startTime -e endTime -i interval + * eg. java com.sun.web20.util.MemCacheUtility server1:12100 server2:12100 -s 300 -e 600 -i 3 + * This will sleep for 300 seconds during ramp up, collect for 600 seconds with an interval of 3 seconds between + * each snapshot. + * @param args String [] + * + */ + public static void main (String[] args) { + + if (args==null || args.length < 4) {//minimum amount of args - one server, -s, -e, -i + System.out.println("Usage: java com.sun.web20.MemCacheUtility server:port [server2:port server3:port] -s startTime -e endTime -i interval"); + System.out.println(" where startTime = ramp up time in seconds. Statistics collection will NOT occur during ramp up time and will sleep for startTime period"); + System.out.println(" endTime = steady State time in seconds. Statistics collection will only occur during the steady state period"); + System.out.println(" interval = time between statistics collection snapshots, in seconds."); + } + + + int startTime = 0; + int endTime = 0; + int intervalTime = 0; + LinkedHashSet serverSet = new LinkedHashSet(); + for (int i = 0; i < args.length; i++) { + if (args[i].startsWith("-s")) { + if (args[i].length() > 2) // -sarg + startTime = Integer.parseInt(args[i].substring(2)) * 1000; + else // -s arg + startTime = Integer.parseInt(args[++i]) * 1000; + } else if (args[i].startsWith("-e")) { + if (args[i].length() > 2) // -earg + endTime = Integer.parseInt(args[i].substring(2)) * 1000; + else // -e arg + endTime = Integer.parseInt(args[++i]) * 1000; + } else if (args[i].startsWith("-i")) { + if (args[i].length() > 2) // -iarg + intervalTime = + Integer.parseInt(args[i].substring(2)) * 1000; + else // -i arg + intervalTime = Integer.parseInt(args[++i])* 1000; + } else if (args[i].contains(":")) {// host:port pair + serverSet.add(args[i]); + } else { // host only. Append default port 11211. + serverSet.add(args[i] + ":11211"); + } + } + + //finished processing all of the args. populate server list + String memCacheServers[] = new String[serverSet.size()]; + memCacheServers = serverSet.toArray(memCacheServers); + + logger.info("Starting memcache stats"); + + //collect only during steady state + MemCacheUtility memCacheUtil = new MemCacheUtility(memCacheServers); + + try { + Timer timer = new Timer(); + MemCacheTask task = new MemCacheTask(memCacheUtil); + timer.scheduleAtFixedRate(task, startTime, intervalTime); + //only print stats for steady state period + Thread.sleep(endTime); + //wake up and stop printing stats + timer.cancel(); + } catch (InterruptedException ex) { + ex.printStackTrace(); + return; + } + } + + /* class for TimerTask */ + + private static class MemCacheTask extends TimerTask { + + private MemCacheUtility memCacheUtility; + + public MemCacheTask(MemCacheUtility memCacheUtil) { + memCacheUtility = memCacheUtil; + + } + + public void run() { + + System.out.println(memCacheUtility.getTemporaryStats()); + + } + + } + + + + +} Added: incubator/olio/workload/rails/trunk/src/com/sun/web20/util/RandomUtil.java URL: http://svn.apache.org/viewvc/incubator/olio/workload/rails/trunk/src/com/sun/web20/util/RandomUtil.java?rev=706345&view=auto ============================================================================== --- incubator/olio/workload/rails/trunk/src/com/sun/web20/util/RandomUtil.java (added) +++ incubator/olio/workload/rails/trunk/src/com/sun/web20/util/RandomUtil.java Mon Oct 20 10:39:16 2008 @@ -0,0 +1,328 @@ +package com.sun.web20.util; + +import com.sun.faban.driver.util.Random; + +import java.io.FileReader; +import java.util.HashSet; +import java.util.logging.Logger; + +public class RandomUtil { + + public static final String[] TIMEZONES = { "ACT", "AET", "AGT", "ART", + "AST", "Africa/Abidjan", "Africa/Accra", "Africa/Addis_Ababa", + "Africa/Algiers", "Africa/Asmera", "Africa/Bamako", "Africa/Bangui", + "Africa/Banjul", "Africa/Bissau", "Africa/Blantyre", + "Africa/Brazzaville", "Africa/Bujumbura", "Africa/Cairo", + "Africa/Casablanca", "Africa/Ceuta", "Africa/Conakry", "Africa/Dakar", + "Africa/Dar_es_Salaam", "Africa/Djibouti", "Africa/Douala", + "Africa/El_Aaiun", "Africa/Freetown", "Africa/Gaborone", + "Africa/Harare", "Africa/Johannesburg", "Africa/Kampala", + "Africa/Khartoum", "Africa/Kigali", "Africa/Kinshasa", "Africa/Lagos", + "Africa/Libreville", "Africa/Lome", "Africa/Luanda", + "Africa/Lubumbashi", "Africa/Lusaka", "Africa/Malabo", "Africa/Maputo", + "Africa/Maseru", "Africa/Mbabane", "Africa/Mogadishu", + "Africa/Monrovia", "Africa/Nairobi", "Africa/Ndjamena", "Africa/Niamey", + "Africa/Nouakchott", "Africa/Ouagadougou", "Africa/Porto-Novo", + "Africa/Sao_Tome", "Africa/Timbuktu", "Africa/Tripoli", "Africa/Tunis", + "Africa/Windhoek", "America/Adak", "America/Anchorage", + "America/Anguilla", "America/Antigua", "America/Araguaina", + "America/Argentina/Buenos_Aires", "America/Argentina/Catamarca", + "America/Argentina/ComodRivadavia", "America/Argentina/Cordoba", + "America/Argentina/Jujuy", "America/Argentina/La_Rioja", + "America/Argentina/Mendoza", "America/Argentina/Rio_Gallegos", + "America/Argentina/San_Juan", "America/Argentina/Tucuman", + "America/Argentina/Ushuaia", "America/Aruba", "America/Asuncion", + "America/Atikokan", "America/Atka", "America/Bahia", "America/Barbados", + "America/Belem", "America/Belize", "America/Blanc-Sablon", + "America/Boa_Vista", "America/Bogota", "America/Boise", + "America/Buenos_Aires", "America/Cambridge_Bay", "America/Campo_Grande", + "America/Cancun", "America/Caracas", "America/Catamarca", + "America/Cayenne", "America/Cayman", "America/Chicago", + "America/Chihuahua", "America/Coral_Harbour", "America/Cordoba", + "America/Costa_Rica", "America/Cuiaba", "America/Curacao", + "America/Danmarkshavn", "America/Dawson", "America/Dawson_Creek", + "America/Denver", "America/Detroit", "America/Dominica", + "America/Edmonton", "America/Eirunepe", "America/El_Salvador", + "America/Ensenada", "America/Fort_Wayne", "America/Fortaleza", + "America/Glace_Bay", "America/Godthab", "America/Goose_Bay", + "America/Grand_Turk", "America/Grenada", "America/Guadeloupe", + "America/Guatemala", "America/Guayaquil", "America/Guyana", + "America/Halifax", "America/Havana", "America/Hermosillo", + "America/Indiana/Indianapolis", "America/Indiana/Knox", + "America/Indiana/Marengo", "America/Indiana/Petersburg", + "America/Indiana/Vevay", "America/Indiana/Vincennes", + "America/Indianapolis", "America/Inuvik", "America/Iqaluit", + "America/Jamaica", "America/Jujuy", "America/Juneau", + "America/Kentucky/Louisville", "America/Kentucky/Monticello", + "America/Knox_IN", "America/La_Paz", "America/Lima", + "America/Los_Angeles", "America/Louisville", "America/Maceio", + "America/Managua", "America/Manaus", "America/Martinique", + "America/Mazatlan", "America/Mendoza", "America/Menominee", + "America/Merida", "America/Mexico_City", "America/Miquelon", + "America/Moncton", "America/Monterrey", "America/Montevideo", + "America/Montreal", "America/Montserrat", "America/Nassau", + "America/New_York", "America/Nipigon", "America/Nome", + "America/Noronha", "America/North_Dakota/Center", + "America/North_Dakota/New_Salem", "America/Panama", + "America/Pangnirtung", "America/Paramaribo", "America/Phoenix", + "America/Port-au-Prince", "America/Port_of_Spain", "America/Porto_Acre", + "America/Porto_Velho", "America/Puerto_Rico", "America/Rainy_River", + "America/Rankin_Inlet", "America/Recife", "America/Regina", + "America/Rio_Branco", "America/Rosario", "America/Santiago", + "America/Santo_Domingo", "America/Sao_Paulo", "America/Scoresbysund", + "America/Shiprock", "America/St_Johns", "America/St_Kitts", + "America/St_Lucia", "America/St_Thomas", "America/St_Vincent", + "America/Swift_Current", "America/Tegucigalpa", "America/Thule", + "America/Thunder_Bay", "America/Tijuana", "America/Toronto", + "America/Tortola", "America/Vancouver", "America/Virgin", + "America/Whitehorse", "America/Winnipeg", "America/Yakutat", + "America/Yellowknife", "Antarctica/Casey", "Antarctica/Davis", + "Antarctica/DumontDUrville", "Antarctica/Mawson", "Antarctica/McMurdo", + "Antarctica/Palmer", "Antarctica/Rothera", "Antarctica/South_Pole", + "Antarctica/Syowa", "Antarctica/Vostok", "Arctic/Longyearbyen", + "Asia/Aden", "Asia/Almaty", "Asia/Amman", "Asia/Anadyr", "Asia/Aqtau", + "Asia/Aqtobe", "Asia/Ashgabat", "Asia/Ashkhabad", "Asia/Baghdad", + "Asia/Bahrain", "Asia/Baku", "Asia/Bangkok", "Asia/Beirut", + "Asia/Bishkek", "Asia/Brunei", "Asia/Calcutta", "Asia/Choibalsan", + "Asia/Chongqing", "Asia/Chungking", "Asia/Colombo", "Asia/Dacca", + "Asia/Damascus", "Asia/Dhaka", "Asia/Dili", "Asia/Dubai", + "Asia/Dushanbe", "Asia/Gaza", "Asia/Harbin", "Asia/Hong_Kong", + "Asia/Hovd", "Asia/Irkutsk", "Asia/Istanbul", "Asia/Jakarta", + "Asia/Jayapura", "Asia/Jerusalem", "Asia/Kabul", "Asia/Kamchatka", + "Asia/Karachi", "Asia/Kashgar", "Asia/Katmandu", "Asia/Krasnoyarsk", + "Asia/Kuala_Lumpur", "Asia/Kuching", "Asia/Kuwait", "Asia/Macao", + "Asia/Macau", "Asia/Magadan", "Asia/Makassar", "Asia/Manila", + "Asia/Muscat", "Asia/Nicosia", "Asia/Novosibirsk", "Asia/Omsk", + "Asia/Oral", "Asia/Phnom_Penh", "Asia/Pontianak", "Asia/Pyongyang", + "Asia/Qatar", "Asia/Qyzylorda", "Asia/Rangoon", "Asia/Riyadh", + "Asia/Riyadh87", "Asia/Riyadh88", "Asia/Riyadh89", "Asia/Saigon", + "Asia/Sakhalin", "Asia/Samarkand", "Asia/Seoul", "Asia/Shanghai", + "Asia/Singapore", "Asia/Taipei", "Asia/Tashkent", "Asia/Tbilisi", + "Asia/Tehran", "Asia/Tel_Aviv", "Asia/Thimbu", "Asia/Thimphu", + "Asia/Tokyo", "Asia/Ujung_Pandang", "Asia/Ulaanbaatar", + "Asia/Ulan_Bator", "Asia/Urumqi", "Asia/Vientiane", "Asia/Vladivostok", + "Asia/Yakutsk", "Asia/Yekaterinburg", "Asia/Yerevan", "Atlantic/Azores", + "Atlantic/Bermuda", "Atlantic/Canary", "Atlantic/Cape_Verde", + "Atlantic/Faeroe", "Atlantic/Jan_Mayen", "Atlantic/Madeira", + "Atlantic/Reykjavik", "Atlantic/South_Georgia", "Atlantic/St_Helena", + "Atlantic/Stanley", "Australia/ACT", "Australia/Adelaide", + "Australia/Brisbane", "Australia/Broken_Hill", "Australia/Canberra", + "Australia/Currie", "Australia/Darwin", "Australia/Hobart", + "Australia/LHI", "Australia/Lindeman", "Australia/Lord_Howe", + "Australia/Melbourne", "Australia/NSW", "Australia/North", + "Australia/Perth", "Australia/Queensland", "Australia/South", + "Australia/Sydney", "Australia/Tasmania", "Australia/Victoria", + "Australia/West", "Australia/Yancowinna", "BET", "BST", "Brazil/Acre", + "Brazil/DeNoronha", "Brazil/East", "Brazil/West", "CAT", "CET", "CNT", + "CST", "CST6CDT", "CTT", "Canada/Atlantic", "Canada/Central", + "Canada/East-Saskatchewan", "Canada/Eastern", "Canada/Mountain", + "Canada/Newfoundland", "Canada/Pacific", "Canada/Saskatchewan", + "Canada/Yukon", "Chile/Continental", "Chile/EasterIsland", "Cuba", + "EAT", "ECT", "EET", "EST", "EST5EDT", "Egypt", "Eire", "Etc/GMT", + "Etc/GMT+0", "Etc/GMT+1", "Etc/GMT+10", "Etc/GMT+11", "Etc/GMT+12", + "Etc/GMT+2", "Etc/GMT+3", "Etc/GMT+4", "Etc/GMT+5", "Etc/GMT+6", + "Etc/GMT+7", "Etc/GMT+8", "Etc/GMT+9", "Etc/GMT-0", "Etc/GMT-1", + "Etc/GMT-10", "Etc/GMT-11", "Etc/GMT-12", "Etc/GMT-13", "Etc/GMT-14", + "Etc/GMT-2", "Etc/GMT-3", "Etc/GMT-4", "Etc/GMT-5", "Etc/GMT-6", + "Etc/GMT-7", "Etc/GMT-8", "Etc/GMT-9", "Etc/GMT0", "Etc/Greenwich", + "Etc/UCT", "Etc/UTC", "Etc/Universal", "Etc/Zulu", "Europe/Amsterdam", + "Europe/Andorra", "Europe/Athens", "Europe/Belfast", "Europe/Belgrade", + "Europe/Berlin", "Europe/Bratislava", "Europe/Brussels", + "Europe/Bucharest", "Europe/Budapest", "Europe/Chisinau", + "Europe/Copenhagen", "Europe/Dublin", "Europe/Gibraltar", + "Europe/Guernsey", "Europe/Helsinki", "Europe/Isle_of_Man", + "Europe/Istanbul", "Europe/Jersey", "Europe/Kaliningrad", "Europe/Kiev", + "Europe/Lisbon", "Europe/Ljubljana", "Europe/London", + "Europe/Luxembourg", "Europe/Madrid", "Europe/Malta", + "Europe/Mariehamn", "Europe/Minsk", "Europe/Monaco", "Europe/Moscow", + "Europe/Nicosia", "Europe/Oslo", "Europe/Paris", "Europe/Prague", + "Europe/Riga", "Europe/Rome", "Europe/Samara", "Europe/San_Marino", + "Europe/Sarajevo", "Europe/Simferopol", "Europe/Skopje", "Europe/Sofia", + "Europe/Stockholm", "Europe/Tallinn", "Europe/Tirane", + "Europe/Tiraspol", "Europe/Uzhgorod", "Europe/Vaduz", "Europe/Vatican", + "Europe/Vienna", "Europe/Vilnius", "Europe/Volgograd", "Europe/Warsaw", + "Europe/Zagreb", "Europe/Zaporozhye", "Europe/Zurich", "GB", "GB-Eire", + "GMT", "GMT0", "Greenwich", "HST", "Hongkong", "IET", "IST", "Iceland", + "Indian/Antananarivo", "Indian/Chagos", "Indian/Christmas", + "Indian/Cocos", "Indian/Comoro", "Indian/Kerguelen", "Indian/Mahe", + "Indian/Maldives", "Indian/Mauritius", "Indian/Mayotte", + "Indian/Reunion", "Iran", "Israel", "JST", "Jamaica", "Japan", + "Kwajalein", "Libya", "MET", "MIT", "MST", "MST7MDT", + "Mexico/BajaNorte", "Mexico/BajaSur", "Mexico/General", + "Mideast/Riyadh87", "Mideast/Riyadh88", "Mideast/Riyadh89", "NET", + "NST", "NZ", "NZ-CHAT", "Navajo", "PLT", "PNT", "PRC", "PRT", "PST", + "PST8PDT", "Pacific/Apia", "Pacific/Auckland", "Pacific/Chatham", + "Pacific/Easter", "Pacific/Efate", "Pacific/Enderbury", + "Pacific/Fakaofo", "Pacific/Fiji", "Pacific/Funafuti", + "Pacific/Galapagos", "Pacific/Gambier", "Pacific/Guadalcanal", + "Pacific/Guam", "Pacific/Honolulu", "Pacific/Johnston", + "Pacific/Kiritimati", "Pacific/Kosrae", "Pacific/Kwajalein", + "Pacific/Majuro", "Pacific/Marquesas", "Pacific/Midway", + "Pacific/Nauru", "Pacific/Niue", "Pacific/Norfolk", "Pacific/Noumea", + "Pacific/Pago_Pago", "Pacific/Palau", "Pacific/Pitcairn", + "Pacific/Ponape", "Pacific/Port_Moresby", "Pacific/Rarotonga", + "Pacific/Saipan", "Pacific/Samoa", "Pacific/Tahiti", "Pacific/Tarawa", + "Pacific/Tongatapu", "Pacific/Truk", "Pacific/Wake", "Pacific/Wallis", + "Pacific/Yap", "Poland", "Portugal", "ROK", "SST", "Singapore", + "SystemV/AST4", "SystemV/AST4ADT", "SystemV/CST6", "SystemV/CST6CDT", + "SystemV/EST5", "SystemV/EST5EDT", "SystemV/HST10", "SystemV/MST7", + "SystemV/MST7MDT", "SystemV/PST8", "SystemV/PST8PDT", "SystemV/YST9", + "SystemV/YST9YDT", "Turkey", "UCT", "US/Alaska", "US/Aleutian", + "US/Arizona", "US/Central", "US/East-Indiana", "US/Eastern", + "US/Hawaii", "US/Indiana-Starke", "US/Michigan", "US/Mountain", + "US/Pacific", "US/Pacific-New", "US/Samoa", "UTC", "Universal", "VST", + "W-SU", "WET", "Zulu"}; + + + // Phone comes as +1 888 999 0000 or +77 66 999 0000 for non-US. + // 50% of the time, do US, 50% non-us. + public static String randomPhone(Random r, StringBuilder b) { + String v = r.makeNString(1, 2); + if (v.length() == 1) { + b.append("+1 "); + v = r.makeNString(3, 3); + b.append(v).append(' '); + } else { + b.append("+").append(v); + v = r.makeNString(2, 2); + b.append(' ').append(v).append(' '); + } + v = r.makeNString(3, 3); + b.append(v).append(' '); + v = r.makeNString(4, 4); + b.append(v); + return b.toString(); + } + + public static String randomTimeZone(Random r) { + return TIMEZONES[r.random(0, TIMEZONES.length - 1)]; + } + + public static StringBuilder randomName(Random r, StringBuilder b, + int minLength, int maxLength) { + if (minLength < 1 || maxLength < minLength) + throw new IllegalArgumentException(); + b.append(r.makeCString(1, 1).toUpperCase()); + b.append(r.makeCString(minLength - 1, maxLength - 1).toLowerCase()); + return b; + } + + /** + * From http://tagsonomy.com/index.php/dynamic-growth-of-tag-clouds/ + + * "It took only 10 users (not quite 1.5% of the current total) before + * the top 3 tags were tagging ontology folksonomy, conveying much + * the same sense, with only the use of tagging instead of tags + * making this different from the current set of 3."
+ * + * In order to achieve a high concentration on the first tags added + * to the system, we use a negative exponential distribution to select + * the tags. + * + * @param r The random value generator + * @param meanRatio The point of mean in the range from 0 to 1 + * @return The randomly selected tag id starting with 1, with a + * high probability of the first tags being selected. + */ + public static int randomTagId(Random r, double meanRatio) { + Logger logger = Logger.getLogger(RandomUtil.class.getName()); + double mean = ScaleFactors.tagCount * meanRatio; + + int selected; + int loops = 0; + + do { + double x = r.drandom(0.0, 1.0); + if (x == 0) + x = 0.05; + + // We use a negative exponential distribution to select the tags. + // The first tags tend to be the most often used. + selected = (int)(mean * -Math.log(x)); + + // if (selected >= ScaleFactors.tagCount) + // logger.warning("Selected: " + selected); + + // However, if we exceed the limit, we do not select the last one. + // We redo the selection instead. + } while (selected >= ScaleFactors.tagCount && loops++ < 10); + + if (loops >= 10) + logger.severe("Exceeded loop limit. Selected:" + + selected + " TagCount: " + ScaleFactors.tagCount); + + // We use the user name mechanism to create the tag names + return ++selected; + } + + /** + * Returns a random tag name. Implicitly calls randomTagId. + * @param r The random value generator + * @return The randomly selected tag name. + */ + public static String randomTagName(Random r) { + return UserName.getUserName(r.random(1, ScaleFactors.tagCount)); + //return UserName.getUserName(randomTagId(r, 0.1)); + } + + /** + * Randomly creates text between length x and y, inclusive that is + * separated by spaces. The words are between 1 and 12 characters long. + */ + public static String randomText(Random r, int x, int y) { + int length = r.random(x, y); + StringBuilder buffer = new StringBuilder(length); + int leftover = length; + while (leftover > 0) { + String word = r.makeCString(1, leftover < 12 ? leftover : 12); + buffer.append(word).append(' '); + leftover -= word.length() + 1; + } + return buffer.toString(); + } + + /** + * Randomly selects an event from the events page. + * @param r The random value generator + * @param eventListPage The page from the response buffer + * @return The selected event id, as a string + */ + public static String randomEvent(Random r, StringBuilder eventListPage) { + String search1 = " eventIdSet = new HashSet(); + for (;;) { + idx = eventListPage.indexOf(search1, idx); + if (idx == -1) + break; + // We skip this the php or jsp, just knowing it is 3 chars + idx += search1.length(); + int endIdx = eventListPage.indexOf("\"", idx); + if (endIdx == -1) + break; + eventIdSet.add(eventListPage.substring(idx, endIdx).trim()); + idx = endIdx; + } + int size = eventIdSet.size(); + if (size == 0) + return null; + + String[] eventIds = new String[size]; + eventIds = eventIdSet.toArray(eventIds); + return eventIds[r.random(0, size - 1)]; + } + + // Main for testing RandomEvent. + public static void main(String[] args) throws Exception { + FileReader reader = new FileReader(args[0]); + StringBuilder page = new StringBuilder(); + char[] readBuffer = new char[8192]; + int readSize; + while ((readSize = reader.read(readBuffer, 0, readBuffer.length)) + != -1) + page.append(readBuffer, 0, readSize); + Random r = new Random(); + String selected = randomEvent(r, page); + System.out.println("Selected: " + selected); + } +} Added: incubator/olio/workload/rails/trunk/src/com/sun/web20/util/ScaleFactors.java URL: http://svn.apache.org/viewvc/incubator/olio/workload/rails/trunk/src/com/sun/web20/util/ScaleFactors.java?rev=706345&view=auto ============================================================================== --- incubator/olio/workload/rails/trunk/src/com/sun/web20/util/ScaleFactors.java (added) +++ incubator/olio/workload/rails/trunk/src/com/sun/web20/util/ScaleFactors.java Mon Oct 20 10:39:16 2008 @@ -0,0 +1,70 @@ +package com.sun.web20.util; + +import java.util.logging.Logger; + +/** + * This static class all the scale factors used for loading the data and + * load generation. + */ +public class ScaleFactors { + + /** The total number of loaded users */ + public static int activeUsers = -1; + public static int users; + public static int events; + public static int tagCount; + + private static Logger logger = + Logger.getLogger(ScaleFactors.class.getName()); + + /** + * Sets the number of users for the run/load. + * @param userCount + */ + public static void setActiveUsers(int userCount) { + if (userCount < 25) { + logger.warning("Trying to load for " + userCount + " concurrent " + + "users which is below the minimum of 25 users. " + + "Adjusting to 25 users"); + userCount = 25; + } + if (activeUsers == -1) { + activeUsers = userCount; + users = activeUsers * 4; + tagCount = getTagCount(users); + events = tagCount * 3; + } + } + + /** + * From http://tagsonomy.com/index.php/dynamic-growth-of-tag-clouds/ + * "As of this writing, a little over 700 users have tagged it, with 450+ + * with 450+ unique tags, roughly two-thirds of which tags were (of course) + * used by one and only one user."
+ * + * This function uses a cumulative half logistic distribution to determine + * the tag growth. We have tested the distribution to be close to the + * quote above. I.e. 175 multi-user tags for 700 users. The quote above + * gives us one-third of 450 which is 150. Close enough. + * + * @param users The users of the data loaded. + * @return The tag count at this users + */ + public static int getTagCount(int users) { + double prob = cumuHalfLogistic(users, 10000); + // We limit to 5000 tags + return (int) Math.round(5000 * prob); + } + + /** + * The cumulative half logistic distribution itself, in the flesh! + * @param x The value on the x axis, in this case the number of users + * @param scale Determines the x-stretch of the curve how far it takes + * for the probability to converge to 1 + * @return The resulting probability or y axis + */ + private static double cumuHalfLogistic(double x, double scale) { + double power = -x / scale; + return (1d - Math.exp(power)) / (1d + Math.exp(power)); + } +} Added: incubator/olio/workload/rails/trunk/src/com/sun/web20/util/Scramble.java URL: http://svn.apache.org/viewvc/incubator/olio/workload/rails/trunk/src/com/sun/web20/util/Scramble.java?rev=706345&view=auto ============================================================================== --- incubator/olio/workload/rails/trunk/src/com/sun/web20/util/Scramble.java (added) +++ incubator/olio/workload/rails/trunk/src/com/sun/web20/util/Scramble.java Mon Oct 20 10:39:16 2008 @@ -0,0 +1,133 @@ +package com.sun.web20.util; + +/* + * Copyright (c) 2000-2004 by Sun Microsystems, Inc. All Rights Reserved. + * This software is the proprietary information of Sun Microsystems, Inc. + * Use is subject to license terms. + * + * $Id: Scramble.java,v 1.1 2007/06/20 19:57:47 akara Exp $ + * + */ + +import java.util.*; + +/** + * Scramble scrambles char sequences in a controlled random fasion. The + * outcome of running the Scramble utility will be the same for the same + * random number generator. Scramble is a utility to generate a code segment + * and is not used in the benchmark itself. + * + * @author Akara Sucharitakul + */ +public class Scramble { + + /** + * The alpha-numeric char sequence. + */ + private static char[] alpha = + {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', + 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '_'}; + /** + * The alpha characters only. A name would start with these. + */ + private static char[] characs = + {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; + + /** + * Generates the char sequences for each length. + * + * @param digits The length of the name to generate the scramble for + * @return The scrambled sequence for each digit + */ + public static char[][] genChars(int digits) { + Random r = new Random(digits); + char[][] result = new char[digits][]; + ArrayList list = new ArrayList(alpha.length); + + // Generate scrambled sequence for first digit. + result[0] = new char[characs.length]; + for (int j = 0; j < characs.length; j++) + list.add(characs[j]); + for (int j = 0; j < characs.length; j++) { + int idx = r.nextInt(list.size()); + result[0][j] = list.remove(idx); + } + + // Generate scrambled sequence for the following digits. + for (int i = 1; i < digits; i++) { + result[i] = new char[alpha.length]; + for (int j = 0; j < alpha.length; j++) + list.add(alpha[j]); + for (int j = 0; j < alpha.length; j++) { + int idx = r.nextInt(list.size()); + result[i][j] = list.remove(idx); + } + } + return result; + } + + /** + * Dumps the generated scramble to standard output in a ready-to-use + * Java array representation. The user will have to redirect the output + * to a Java class by him/herself. + * + * @param arrays The scramble for each length + */ + public static void dumpArrays(char[][][] arrays) { + StringBuilder buffer = new StringBuilder(8096); + buffer.append("private static final char[][][] scramble = {"); + for (int i = 0; i < arrays.length; i++) { + buffer.append('{'); + for (int j = 0; j < arrays[i].length; j++) { + buffer.append('{'); + for (int k = 0; k < arrays[i][j].length; k++) { + buffer.append("'"); + buffer.append(arrays[i][j][k]); + if (k < arrays[i][j].length - 1) { + buffer.append("',"); + if (buffer.length() > 70) { + System.out.println(buffer); + buffer.setLength(0); + } else { + buffer.append(' '); + } + } else { + buffer.append("'"); + } + } + buffer.append("}"); + if (j < arrays[i].length - 1) { + buffer.append(','); + System.out.println(buffer); + buffer.setLength(0); + } + } + buffer.append("}"); + if (i < arrays.length - 1) { + buffer.append(','); + System.out.println(buffer); + buffer.setLength(0); + } + } + buffer.append("};"); + System.out.println(buffer); + } + + /** + * The main method to drive the scrambling. + * + * @param args ignored + */ + public static void main(String[] args) { + char[][][] results = new char[8][][]; + char[][] nullArray = new char[1][1]; + nullArray[0][0] = '0'; + for (int i = 0; i < 2; i++) + results[i] = nullArray; + for (int i = 2; i < 8; i++) + results[i] = genChars(i + 1); + dumpArrays(results); + } +} Added: incubator/olio/workload/rails/trunk/src/com/sun/web20/util/UserName.java URL: http://svn.apache.org/viewvc/incubator/olio/workload/rails/trunk/src/com/sun/web20/util/UserName.java?rev=706345&view=auto ============================================================================== --- incubator/olio/workload/rails/trunk/src/com/sun/web20/util/UserName.java (added) +++ incubator/olio/workload/rails/trunk/src/com/sun/web20/util/UserName.java Mon Oct 20 10:39:16 2008 @@ -0,0 +1,249 @@ +package com.sun.web20.util; +/* + * Copyright (c) 2000-2004 by Sun Microsystems, Inc. All Rights Reserved. + * This software is the proprietary information of Sun Microsystems, Inc. + * Use is subject to license terms. + * + * $Id: UserName.java,v 1.1 2007/06/20 19:57:47 akara Exp $ + * + */ + +import java.util.HashSet; +import java.io.*; + +/** + * The user utility creates usernames and passwords from user ids. + * It uses the scrambled character sequences generated by the Scramble + * class. The username length distribution are controlled so 5% of the + * users have 3 letter names, 8% of the users have 4 letter names, + * 17% of the users have 5 letter names, 25% of the users have 6 letter + * names, 24% of the users have 7 letter names, and 21% of the users + * have 8 letter names. This ratio is based on Sun's own users that + * DO NOT use standard names such as ab123456.

+ * At the current state, this utility can generate 711,880 non-duplicate + * names. + * + * @author Akara Sucharitakul + */ +public class UserName { + + private static final char[][][] scramble = {{{'0'}}, {{'0'}}, + {{'s', 'k', 'u', 'p', 'n', 't', 'j', 'z', 'b', 'g', 'q', 'l', 'w', 'd', + 'v', 'y', 'm', 'f', 'x', 'i', 'c', 'h', 'r', 'o', 'e', 'a'}, + {'w', '_', 'q', 'l', 'z', 'h', 'j', 'k', '1', '6', 'e', '5', '4', 't', 'b', + '2', 'p', 'r', 'g', 'm', '8', '3', 'c', 'u', '7', 'v', 'i', 'y', 'o', 'a', + 'x', 'n', '0', '9', 's', 'd', 'f'}, + {'8', '7', '2', 'm', 'l', 's', 'q', '6', '1', 'j', 'b', 'n', '3', 'i', '_', + 'w', 'a', 'z', '4', 'x', 'e', 'f', '9', 'c', 'g', 't', 'h', '0', 'd', 'y', + 'k', 'v', 'u', 'p', 'r', '5', 'o'}}, + {{'q', 'c', 'r', 'p', 'm', 'o', 'n', 'y', 't', 'x', 'a', 'k', 'l', 's', + 'e', 'g', 'z', 'h', 'w', 'j', 'v', 'u', 'b', 'f', 'i', 'd'}, + {'0', 'g', 'l', '8', 'f', 'p', 'x', 'q', 'm', '6', 's', 'y', 'c', '2', '7', + 'e', '4', '5', 'v', 'b', 'z', 'i', 'd', 'h', '1', 'n', '3', '9', 'w', 'o', + '_', 'r', 'j', 't', 'k', 'u', 'a'}, + {'5', 'e', '0', '4', 'j', 'd', 'a', 'b', 'g', 'w', 'n', 'v', '3', 'u', '9', + 'x', 'i', 'c', 'h', 'q', 'y', 'r', '_', 'm', 't', '6', '8', '2', 'l', 'k', + 'p', '7', 's', 'o', 'z', 'f', '1'}, + {'r', 'v', 'g', 'a', '0', 'w', 't', '7', '5', '_', 'o', '6', 'f', 'z', 'j', + '2', 'c', 'e', '4', '1', 's', 'y', 'p', '8', 'n', 'u', '3', 'l', 'k', 'h', + 'd', 'b', 'q', 'x', 'i', '9', 'm'}}, + {{'p', 's', 'c', 'o', 'r', 't', 'u', 'i', 'k', 'q', 'v', 'e', 'f', 'x', + 'n', 'j', 'y', 'w', 'z', 'd', 'a', 'b', 'g', 'm', 'l', 'h'}, + {'i', 'f', 'w', '0', '5', 'p', '7', 'o', '6', 'z', '8', 'h', 't', '2', '4', + 'u', 'a', 'x', 'r', 's', '9', 'v', 'k', 'j', 'e', 'g', '3', 'q', 'm', '1', + 'l', 'y', 'd', 'n', '_', 'c', 'b'}, + {'n', 'z', 'q', 'd', '8', '1', '5', 'o', 'b', 'u', 'e', 'a', 'c', '2', 'y', + 'w', 'k', '9', 'i', '7', 'h', '_', '3', '0', 'j', 'g', 'l', 'm', 'p', 's', + 'v', 't', '4', 'x', '6', 'f', 'r'}, + {'f', 'd', '5', 't', 'j', 'e', 'h', 'q', 'u', 'c', 'b', 'w', 'l', 'k', 'a', + 'n', 'r', 'm', '1', 'o', '_', 'y', '6', 's', '7', 'g', 'i', '2', '0', 'p', + 'x', 'v', '3', '4', 'z', '9', '8'}, + {'4', 'm', 'q', 'o', '9', 'v', 'z', 'w', '7', '6', '5', '8', 'f', 'g', 'd', + 'a', 'p', 'j', '1', '_', 'y', '3', 'h', 'r', 't', 'c', 'n', 'l', '0', 'e', + 'k', 'b', 'x', 'i', '2', 's', 'u'}}, + {{'r', 'b', 'd', 'm', 'f', 't', 'x', 'e', 'i', 'o', 's', 'p', 'a', 'l', + 'g', 'h', 'n', 'w', 'z', 'q', 'u', 'v', 'k', 'j', 'c', 'y'}, + {'h', 'i', 'b', 'f', '2', 'o', 'd', 'u', '7', '9', 'w', 'v', 'j', '3', '6', + 'g', 'z', 'p', 'n', '8', 'y', 'k', 'x', 'q', '5', 's', 't', 'a', '1', '4', + '0', 'e', 'c', 'm', 'r', 'l', '_'}, + {'y', '3', 'w', 'h', 'v', 'u', 'e', 'q', 'm', 'z', '9', 'x', 'k', '7', 'p', + 'r', 't', 'n', '4', 'f', 'o', '2', '5', 'i', 'l', 'a', '6', '8', 'g', '1', + '_', 'b', 'd', '0', 's', 'j', 'c'}, + {'6', 'c', 'f', '3', '2', 'v', 'm', 'l', 'x', 'k', 'e', '_', '7', 'a', 's', + '0', 'j', 'n', 'd', 'z', 'u', '4', 't', 'o', 'g', '5', 'w', 'h', 'p', 'b', + 'i', '1', 'q', '8', 'r', 'y', '9'}, + {'i', 'e', 'p', 'g', 'h', 'b', '1', 'a', 'x', 'm', 'o', '7', 'l', 'u', 'z', + 'w', '0', '8', '9', 'd', 'f', '2', '_', '5', 'c', 'n', 'r', '6', 'j', 'v', + 'q', 't', 's', '4', 'k', '3', 'y'}, + {'t', 'n', 'v', 'g', 's', 'j', 'p', 'l', 'b', '8', 'd', '_', 'q', 'u', 'z', + '2', '5', '7', 'x', 'i', 'o', 'r', '9', '0', 'f', 'w', 'k', '6', '4', '3', + 'e', 'c', 'a', 'h', 'm', '1', 'y'}}, + {{'q', 'o', 'x', 'j', 'g', 'r', 'k', 'p', 'a', 'e', 'i', 'w', 'u', 'n', + 's', 'f', 'c', 'b', 'z', 'y', 't', 'v', 'm', 'h', 'l', 'd'}, + {'z', 'g', 'm', '7', 'r', 'l', 'o', 'q', 't', '0', '9', 'b', 'w', '3', '2', + 'y', '1', 'e', 'p', 's', '6', 'x', '5', 'v', 'i', 'n', '_', '4', 'a', 'k', + 'u', 'f', 'c', 'd', 'h', 'j', '8'}, + {'1', '9', 'c', 't', 'p', 'm', 'e', '5', 'f', 'y', 'r', 'g', 'w', 'j', 'i', + 'x', '3', 'u', '8', '6', 'd', 'k', 's', '4', 'b', 'l', 'h', 'q', 'n', '7', + '2', 'z', 'a', 'o', '0', 'v', '_'}, + {'g', 'v', 'r', 'l', 'h', 'a', '4', '0', 'k', '_', '2', 'j', 'b', 't', 'p', + 'i', 'z', '5', '7', 'm', '3', '1', 'w', 'e', '6', 'u', 'f', 'y', '9', 'n', + 'x', 'o', 'c', 'd', 's', 'q', '8'}, + {'q', 'u', '0', '4', 'f', 'j', 'r', 'w', '8', '9', 't', 'k', 'h', '2', 'i', + 'b', 'n', 'g', 'z', 'x', '3', 'd', 'e', 'a', 's', '6', '1', '5', 'v', 'o', + '_', 'l', 'm', 'y', 'c', 'p', '7'}, + {'b', '7', 'x', 'r', 'a', '8', 'z', 'm', 'q', 'i', 't', 'v', 'c', 'd', '6', + 'j', 'k', 'y', '0', '9', 'f', '3', '2', 'h', 's', 'w', 'l', '1', 'u', '_', + 'g', 'o', 'e', 'n', 'p', '5', '4'}, + {'v', 'j', '1', 'n', '8', 's', '_', 'q', 'u', 'e', '3', 'c', 'o', '9', 'm', + 'h', 't', '0', 'f', '4', 'd', 'r', '5', 'x', '2', '6', 'w', 'a', 'i', 'z', + '7', 'b', 'l', 'k', 'g', 'y', 'p'}}, + {{'o', 'g', 'l', 'k', 'e', 'q', 'r', 'p', 't', 'w', 'u', 'h', 'j', 'a', + 'i', 'v', 'd', 'y', 'z', 'b', 'c', 'm', 'x', 'n', 'f', 's'}, + {'3', 'y', 'f', 't', '6', 'q', 'z', 'r', 'b', '1', 'j', '0', '7', '2', '_', + 'a', 'g', '9', '4', 'l', 'v', 'd', 'c', 'm', 'o', 'i', 'k', '8', '5', 'x', + 'w', 'n', 'h', 'u', 'p', 'e', 's'}, + {'d', 'v', 'l', 'b', 'j', '5', 'y', '8', 'o', 'p', '0', 'q', 'x', 'u', 's', + 'w', '7', '3', 'h', 't', '_', 'n', '2', 'c', 'm', 'r', '6', 'g', 'k', 'z', + 'e', '1', 'f', '9', '4', 'i', 'a'}, + {'a', 'f', 'y', 'o', 'w', 'z', 'b', 'i', 'd', '7', '_', 'm', 's', 'p', '1', + '4', 'x', 'l', 'r', '9', 'j', 'q', 'k', 'v', '6', 'g', '3', 't', 'h', 'e', + '2', '0', 'n', '5', '8', 'u', 'c'}, + {'r', 'c', 'q', 'x', '1', 'a', 'u', '7', 'k', '8', 'p', '0', '9', 'f', 'j', + 'n', 'b', '3', 'z', 'o', '_', 'd', 'v', '4', 'g', 'i', 's', 'e', 'w', 'y', + '2', 'm', 't', '5', 'h', '6', 'l'}, + {'s', '_', '2', 'v', 'z', 'f', '1', '7', 'k', 'o', 'd', '6', '8', 'j', 'i', + 'q', '0', 'x', 'a', 'm', 'r', 't', 'w', 'h', 'y', 'n', 'l', 'u', 'c', 'p', + 'g', '4', '9', '3', '5', 'e', 'b'}, + {'u', '5', 'h', 'x', 'y', 'a', '4', '8', 'z', 'i', 'g', 's', '2', 'n', 'p', + 'b', 'q', 'o', '6', '1', '0', 'w', 'e', '3', '_', 'j', 'v', '9', 'k', 'm', + 'd', 'r', 't', '7', 'f', 'l', 'c'}, + {'t', 'z', 'n', 'y', '6', 'm', 'i', 'w', 'c', '2', 'f', 'q', 'e', 'h', '_', + 'v', 'j', '9', '0', '8', 's', '5', 'g', 'd', 'p', 'l', '4', 'u', 'o', '1', + 'k', '3', 'r', 'b', 'a', 'x', '7'}}}; + + + // Note that these sum up to 100 + private static final int[] length_percent = { 0, 0, 5, 8, 17, 25, 24, 21 }; + + private static int[] selector = new int[length_percent.length]; + + static { + selector[0] = length_percent[0]; + for (int i = 1; i < selector.length; i++) + selector[i] = selector[i - 1] + length_percent[i]; + } + + /** + * Obtains the unique user name for the given user id. + * @param id The user id + * @return The unique user name + */ + public static String getUserName(long id) { + + // Since id starts with 1, we have to shift it to start with 0 for + // our operations. + --id; + + // We divide the ids into sets, each set has 100 users. + int setId = (int) (id / 100); + + // Then we obtain the per-set id 0..99 + int psid = (int) (id % 100); + + // Here we reverse odd ids to ovid cluttering of shorter names + // in the lower range and longer ones in the upper range of each + // 100. + if (psid % 2 == 1) + psid = 100 - psid; + + // For selection, we do not want to make the same name lengths + // contigous. So we switch the digits on psid. + psid = (psid % 10) * 10 + (psid / 10); + + + // This outcoming psid is used for digit selection. + + // Next, choose the length. + int lengthSequence = 0; // This is the sequence number for the psid + // having this length within these 100 names. + int len; // For now, pretend 0 is OK, but we'll shift is back to 1. + for (len = 0; len < selector.length; len++) { + if (psid < selector[len]) { + if (len == 0) + lengthSequence = psid; + else + lengthSequence = psid - selector[len - 1]; + break; + } + } + // Here we shift it back so len is from 1 to whatever. + ++len; + + // Now as we know the id, psid, and the name length to use, + // we have to generate the name. + char[] name = new char[len]; + int[] offset = new int[len]; + + // The lengthId is the unique identifier for this length and is the + // value we use to get the name. + int lengthId = length_percent[len - 1] * setId + lengthSequence; + + // Now we calculate the initial offset into the scrambled chars + // using last digit first. + for (int i = 0; i < len; i++) { + offset[i] = lengthId % scramble[len - 1][i].length; + lengthId /= scramble[len - 1][i].length; + } + + // The first offset is now taken as is. + name[0] = scramble[len - 1][0][offset[0]]; + + for (int i = 1; i < len; i++) { + // We adjust the offset once again to avoid same name lenghts + // to have many of the same characters. We use the previous + // offset to step up the current offset. + offset[i] = (offset[i] + offset[i - 1]) % + scramble[len - 1][i].length; + // And finally we assign the rest of the name. + name[i] = scramble[len - 1][i][offset[i]]; + } + + return new String(name); + } + + /** + * The main method is used for testing user name generation. + * It prints out all the user names generated and the + * percentages of user names for each length. It also + * reports duplicate users, whenever found. + */ + public static void main (String[] args) throws Exception { + long limit = Long.parseLong(args[0]) * 100; + FileWriter tmpfile; + if (args.length <= 1) + tmpfile = new FileWriter("/tmp/users.txt"); + else tmpfile = new FileWriter(args[1]); + + int[] nameLength = new int[8]; + HashSet set = new HashSet((int) (limit - 1)); + + for (long i = 1; i <= limit; i++) { + String name = getUserName(i); + System.out.println("User " + i + ": " + name); + tmpfile.write(name + "\n"); + ++nameLength[name.length() - 1]; + if (!set.add(name)) + System.out.println("Alert! Duplicate name: " + name); + } + tmpfile.close(); + long count = 0; + for (int i = 0; i < nameLength.length; i++) { + count += nameLength[i]; + } + for (int i = 0; i < nameLength.length; i++) { + System.out.println("Length " + (i + 1) + ", count " + + nameLength[i] + ", " + (100d * nameLength[i]/count) + "%"); + } + } +}