incubator-olio-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject svn commit: r745184 [2/3] - in /incubator/olio/workload/php/trunk: ./ bin/ deploy/ src/org/ src/org/apache/ src/org/apache/olio/ src/org/apache/olio/workload/ src/org/apache/olio/workload/driver/ src/org/apache/olio/workload/fsloader/ src/org/apache/ol...
Date Tue, 17 Feb 2009 18:25:44 GMT
Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/EventTag.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/EventTag.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/EventTag.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/EventTag.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id: EventTag.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader;
+
+import com.sun.faban.driver.util.Random;
+import org.apache.olio.workload.util.RandomUtil;
+import org.apache.olio.workload.loader.framework.Loadable;
+import org.apache.olio.workload.loader.framework.ThreadConnection;
+import org.apache.olio.workload.loader.framework.ThreadResource;
+
+import java.util.LinkedHashSet;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * The Tag-Event relationship loader.
+ */
+public class EventTag extends Loadable{
+
+    // We use on average of 3.5 tags per event. Random 1..6 tags.
+    // Once we know the tag count, we have to select tags.
+
+    private static final String STATEMENT = "insert into " +
+            "SOCIALEVENTTAG_SOCIALEVENT " +
+            "(socialeventtagid, socialeventid) values (?, ?)";
+
+    static Logger logger = Logger.getLogger(EventTag.class.getName());
+
+    int eventId;
+    int [] tagIds;
+
+    public String getClearStatement() {
+        return "truncate table SOCIALEVENTTAG_SOCIALEVENT";
+    }
+
+    public void prepare() {
+        eventId = getSequence();
+        ++eventId;
+        ThreadResource tr = ThreadResource.getInstance();
+        Random r = tr.getRandom();
+        int numTags = r.random(1, 7); // Avg is 4 tags per event
+        LinkedHashSet<Integer> tagSet = new LinkedHashSet<Integer>(numTags);
+        for (int i = 0; i < numTags; i++)
+            while (!tagSet.add(RandomUtil.randomTagId(r, 0.1d)));
+
+        tagIds = new int[tagSet.size()];
+        int idx = 0;
+        for (int tagId : tagSet)
+            tagIds[idx++] = tagId;
+    }
+
+
+    public void load() {
+        ThreadConnection c = ThreadConnection.getInstance();
+        try {
+            for (int tagId : tagIds) {
+                PreparedStatement s = c.prepareStatement(STATEMENT);
+                s.setInt(1, tagId);
+                s.setInt(2, eventId);
+                c.addBatch();
+            }
+        } catch (SQLException e) {
+            logger.log(Level.SEVERE, e.getMessage(), e);
+        }
+    }
+}

Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Friends.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Friends.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Friends.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Friends.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id: Friends.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader;
+
+import com.sun.faban.driver.util.Random;
+import org.apache.olio.workload.util.ScaleFactors;
+import org.apache.olio.workload.util.UserName;
+import org.apache.olio.workload.loader.framework.Loadable;
+import org.apache.olio.workload.loader.framework.ThreadConnection;
+import org.apache.olio.workload.loader.framework.ThreadResource;
+
+import java.util.LinkedHashSet;
+import java.util.logging.Logger;
+import java.util.logging.Level;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Friends loader
+ */
+public class Friends extends Loadable {
+    // We use on average of 15 friends. Random 2..28 Friends.
+
+    private static final String STATEMENT = "insert into PERSON_PERSON " +
+            "(Person_username, friends_username,is_accepted) values (?, ?, ?)";
+
+    static Logger logger = Logger.getLogger(Friends.class.getName());
+
+    String userName;
+    String[] friends;
+
+    public String getClearStatement() {
+        return "truncate table PERSON_PERSON";
+    }
+
+    public void prepare() {
+        int id = getSequence();
+        ++id;
+        ThreadResource tr = ThreadResource.getInstance();
+        Random r = tr.getRandom();
+        userName = UserName.getUserName(id);
+        int count = r.random(2, 28);
+
+        LinkedHashSet<Integer> friendSet = new LinkedHashSet<Integer>(count);
+        for (int i = 0; i < count; i++) {
+            int friendId;
+            do { // Prevent friend to be the same user.
+                friendId = r.random(1, ScaleFactors.users);
+            } while (friendId == id || !friendSet.add(friendId));
+        }
+
+        friends = new String[friendSet.size()];
+        int idx = 0;
+        for (int friendId : friendSet)
+            friends[idx++] = UserName.getUserName(friendId);
+    }
+
+    public void load() {
+        ThreadConnection c = ThreadConnection.getInstance();
+        try {
+            int alternate = 0;
+            for (String friend : friends) {
+                PreparedStatement s = c.prepareStatement(STATEMENT);
+                s.setString(1, userName);
+                s.setString(2, friend);
+                if (alternate%2 == 0){
+                    s.setInt(3, 0);
+                }else{
+                    s.setInt(3, 1);
+                }
+                alternate++;
+                c.addBatch();
+            }
+        } catch (SQLException e) {
+            logger.log(Level.SEVERE, e.getMessage(), e);
+        }
+    }
+}

Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/LoadController.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/LoadController.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/LoadController.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/LoadController.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,76 @@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id: LoadController.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader;
+
+import org.apache.olio.workload.util.ScaleFactors;
+
+import java.util.logging.Logger;
+
+import static org.apache.olio.workload.loader.framework.Loader.*;
+
+public class LoadController {
+
+    static Logger logger = Logger.getLogger(LoadController.class.getName());
+
+    public static void main(String[] args) throws Exception {
+        setJDBCDriverClassName(args[0]);
+        setConnectionURL(args[1]);
+        ScaleFactors.setActiveUsers(Integer.parseInt(args[2]));
+
+        // Clear the database
+        clear(Person.class);
+        clear(Friends.class);
+        clear(Address.class);
+        clear(Tag.class);
+        clear(SocialEvent.class);
+        clear(EventTag.class);
+        clear(Attendees.class);
+        clear(Comments.class);
+        logger.info("Done clearing database tables.");
+
+        // load person, friends, and addresses
+        load(Person.class, ScaleFactors.users);
+        load(Friends.class, ScaleFactors.users);
+        load(Address.class, ScaleFactors.users);
+
+        // load tags
+        load(Tag.class, ScaleFactors.tagCount);
+
+        // load events and all relationships to events
+        load(SocialEvent.class, ScaleFactors.events);
+        load(EventTag.class, ScaleFactors.events);
+        load(Attendees.class, ScaleFactors.events);
+        load(Comments.class, ScaleFactors.events);
+
+        waitProcessing();
+        logger.info("Done data creation.");
+
+        // Now we need to check that all loading is done.
+        shutdown();
+        logger.info("Done data loading.");
+
+        // We use a new set of connections and thread pools for postLoad.
+        // This is to ensure all load tasks are done before this one starts.
+        postLoad(Tag.class);
+        shutdown();
+        logger.info("Done post-load.");
+        System.exit(0); // Signal successful loading.
+    }
+}

Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Person.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Person.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Person.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Person.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id: Person.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader;
+
+import com.sun.faban.driver.util.Random;
+import org.apache.olio.workload.util.UserName;
+import org.apache.olio.workload.util.RandomUtil;
+import org.apache.olio.workload.util.ScaleFactors;
+import org.apache.olio.workload.loader.framework.Loadable;
+import org.apache.olio.workload.loader.framework.ThreadConnection;
+import org.apache.olio.workload.loader.framework.ThreadResource;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+
+/**
+ * Person loader
+ */
+public class Person extends Loadable {
+
+    private static final String STATEMENT = "insert into PERSON (username, " +
+            "password, firstname, lastname, email, telephone, imageurl, " +
+            "imagethumburl, summary, timezone, ADDRESS_addressid)" +
+            "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+    static Logger logger = Logger.getLogger(Person.class.getName());
+
+    String[] fields = new String[10];
+    int addressId;
+
+    public String getClearStatement() {
+        return "truncate table PERSON";
+    }
+
+    public void prepare() {
+        int id = getSequence();
+        ++id;
+        ThreadResource tr = ThreadResource.getInstance();
+        Random r = tr.getRandom();
+        StringBuilder b = tr.getBuffer();
+        fields[0] = UserName.getUserName(id);
+        fields[1] = String.valueOf(id);
+        fields[2] = RandomUtil.randomName(r, b, 2, 12).toString();
+        b.setLength(0);
+        fields[3] = RandomUtil.randomName(r, b, 5, 15).toString();
+        fields[4] = r.makeCString(3, 10);
+        fields[4] = fields[2] + '_' + fields[3] + '@' + fields[4] + ".com";
+        b.setLength(0);
+        fields[5] = RandomUtil.randomPhone(r, b);
+        fields[6] = "p" + id + ".jpg";
+        fields[7] = "p" + id + "t.jpg";
+        fields[8] = RandomUtil.randomText(r, 250, 2500);
+        fields[9] = "PST";
+        addressId = r.random(1, ScaleFactors.users);
+    }
+
+    public void load() {
+        ThreadConnection c = ThreadConnection.getInstance();
+        try {
+            PreparedStatement s = c.prepareStatement(STATEMENT);
+            for (int i = 0; i < fields.length; i++)
+                s.setString(i + 1, fields[i]);
+            s.setInt(11, addressId);
+            c.addBatch();
+        } catch (SQLException e) {
+            logger.log(Level.SEVERE, e.getMessage(), e);
+        }
+    }
+}

Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/SocialEvent.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/SocialEvent.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/SocialEvent.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/SocialEvent.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id: SocialEvent.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader;
+
+import com.sun.faban.driver.util.Random;
+import org.apache.olio.workload.util.UserName;
+import org.apache.olio.workload.util.RandomUtil;
+import org.apache.olio.workload.util.ScaleFactors;
+import org.apache.olio.workload.loader.framework.Loadable;
+import org.apache.olio.workload.loader.framework.ThreadConnection;
+import org.apache.olio.workload.loader.framework.ThreadResource;
+
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.text.DateFormat;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * SocialEvent loader.
+ */
+public class SocialEvent extends Loadable {
+
+    public static final Date BASE_DATE = new Date(System.currentTimeMillis());
+
+    private static final String STATEMENT = "insert into SOCIALEVENT " +
+            "(title, description, submitterUserName, imageurl, " +
+            "imagethumburl, literatureurl, telephone, timezone, " +
+            "eventtimestamp, eventdate,summary, createdtimestamp, ADDRESS_addressid, " +
+            "totalscore, numberofvotes, disabled) " +
+            "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+
+    private static final String[] EVT_MINUTES = { "00", "15", "30", "45" };
+
+    static Logger logger = Logger.getLogger(SocialEvent.class.getName());
+
+    String[] fields = new String[11];
+    Date createdTimestamp;
+    int[] ifields = new int[4];
+
+    public String getClearStatement() {
+        return "truncate table SOCIALEVENT";
+    }
+
+    public void prepare() {
+        int id = getSequence();
+        ++id;
+        ThreadResource tr = ThreadResource.getInstance();
+        Random r = tr.getRandom();
+        StringBuilder buffer = tr.getBuffer();
+        fields[0] = RandomUtil.randomText(r, 15, 20); //title
+        fields[1] = RandomUtil.randomText(r, 50, 495); // description
+        fields[2] = UserName.getUserName(r.random(1, ScaleFactors.users));
+        fields[3] = "e" + id + ".jpg"; // imageurl
+        fields[4] = "e" + id + "t.jpg"; // imagethumburl
+        fields[5] = "e" + id + "l.pdf"; //literatureurl
+        fields[6] = RandomUtil.randomPhone(r, buffer); //phone
+        fields[7] = RandomUtil.randomTimeZone(r); // timezone
+        DateFormat dateFormat = tr.getDateFormat(); // eventtimestamp
+        String eventDate = dateFormat.format(
+                    r.makeDateInInterval(BASE_DATE, 0, 540));
+        int eventHr = r.random(7, 21);
+        String eventMin = EVT_MINUTES[r.random(0, 3)];  // eventtimestamp
+        fields[8] = String.format("%s %02d:%s:00",
+                                            eventDate, eventHr, eventMin);
+        fields[9] = eventDate; // eventdate
+        fields[10] = RandomUtil.randomText(r, 50, 90); //summary
+        createdTimestamp = r.makeDateInInterval( //createdtimestamp
+                BASE_DATE, -540, 0);
+        ifields[0] = r.random(1, ScaleFactors.users); // addressId
+        // The rest is initialized to 0 anyway, leave it that way.
+    }
+
+    public void load() {
+        ThreadConnection c = ThreadConnection.getInstance();
+        try {
+            PreparedStatement s = c.prepareStatement(STATEMENT);
+            for (int i = 0; i < fields.length; i++)
+                if (fields[i] != null)
+                    s.setString(i + 1, fields[i]);
+                else
+                    s.setNull(i + 1, Types.VARCHAR);
+            s.setDate(12, createdTimestamp);
+            for (int i = 0; i < ifields.length; i++)
+                s.setInt(13 + i, ifields[i]);
+            c.addBatch();
+        } catch (SQLException e) {
+            logger.log(Level.SEVERE, e.getMessage(), e);
+        }
+    }
+}

Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Tag.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Tag.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Tag.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/Tag.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id: Tag.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader;
+
+import org.apache.olio.workload.util.UserName;
+import org.apache.olio.workload.loader.framework.Loadable;
+import org.apache.olio.workload.loader.framework.Loader;
+import org.apache.olio.workload.loader.framework.ThreadConnection;
+
+import java.util.logging.Logger;
+import java.util.logging.Level;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * The tag loader.
+ */
+public class Tag extends Loadable {
+
+    // Note that the tag id in the database is autoincrement and may
+    // not coincide with this tag id/name when using multi-thread loading.
+    private static final String STATEMENT = "insert into SOCIALEVENTTAG " +
+            "(tag, refcount) values (?, ?)";
+
+    static Logger logger = Logger.getLogger(Tag.class.getName());
+
+    String tag;
+
+    public String getClearStatement() {
+        return "truncate table SOCIALEVENTTAG";
+    }
+
+    public void prepare() {
+        int id = getSequence();
+        ++id;
+        tag = UserName.getUserName(id);
+    }
+
+
+    public void load() {
+        ThreadConnection c = ThreadConnection.getInstance();
+        try {
+            PreparedStatement s = c.prepareStatement(STATEMENT);
+            s.setString(1, tag);
+            s.setInt(2, 0); // Initialize it to 0 first, count and add later.
+            c.addBatch();
+        } catch (SQLException e) {
+            logger.log(Level.SEVERE, e.getMessage(), e);
+            Loader.increaseErrorCount();
+        }
+    }
+
+    /**
+     * For tags, we won't know the refcount till all the data is loaded.
+     * So we update the table at postload.
+     */
+    public void postLoad() {
+        ThreadConnection c = ThreadConnection.getInstance();
+        try {
+            c.prepareStatement("update SOCIALEVENTTAG set refcount = " +
+                    "(select count(*) from SOCIALEVENTTAG_SOCIALEVENT " +
+                    "where socialeventtagid = " +
+                    "SOCIALEVENTTAG.socialeventtagid)");
+            c.executeUpdate();
+        } catch (SQLException e) {
+            logger.log(Level.SEVERE, e.getMessage(), e);
+        }
+
+
+    }
+}

Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loadable.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loadable.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loadable.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loadable.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id: Loadable.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader.framework;
+
+public abstract class Loadable {
+
+    // Sequence is set by the pool.
+    int sequence;
+
+    protected Loader loader = Loader.getInstance(getClass());
+    LoadablePool<? extends Loadable> pool;
+
+    /**
+     * Obtains the sequence, starting from 0, of this loader.
+     *
+     * @return The sequence of this loadable.
+     */
+    protected int getSequence() {
+        return sequence;
+    }
+
+    public abstract String getClearStatement();
+
+    public abstract void prepare();
+
+    public abstract void load();
+
+    public void postLoad() {
+        // Empty. We do not make it abstract.
+        //  A majority of LoadObjects do not need this.
+    }
+}

Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/LoadablePool.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/LoadablePool.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/LoadablePool.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/LoadablePool.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id$
+ */
+package org.apache.olio.workload.loader.framework;
+
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ *
+ * @author akara
+ */
+public class LoadablePool<T extends Loadable> {
+
+    private static Logger logger = Logger.getLogger(LoadablePool.class.getName());
+    LinkedBlockingDeque<T> pool = new LinkedBlockingDeque<T>();
+    int sequence;
+    int count = 0;
+    int size;
+    Class<T> clazz;
+
+    public LoadablePool(int size, Class<T> clazz) {
+        this.size = size;
+        this.clazz = clazz;
+    }
+
+    public T getLoadable() throws Exception {
+        T loadable = pool.poll();
+        if (loadable == null) {
+            if (count < size) {
+                loadable = clazz.newInstance();
+                loadable.pool = this;
+                ++count;
+            } else {
+                for (;;) {
+                    try {
+                        loadable = pool.take();
+                        break;
+                    } catch (InterruptedException ex) {
+                        logger.log(Level.WARNING, "getLoader interrupted", ex);
+                    }
+                }
+            }
+        }
+        loadable.sequence = sequence++;
+        return loadable;
+    }
+
+    @SuppressWarnings("unchecked")
+    public void putLoader(Loadable loadable) {
+        for (;;) {
+            try {
+                // User a LIFO model to keep the hot objects in cache.
+                pool.putFirst((T) loadable);
+                break;
+            } catch (InterruptedException ex) {
+                logger.log(Level.WARNING, "putLoader interrupted!", ex);
+            }
+        }
+    }
+}

Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loader.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loader.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loader.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/Loader.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,343 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id: Loader.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader.framework;
+
+import java.sql.SQLException;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.ArrayList;
+
+/**
+ * The loader, one instance per Loadable type loaded, is responsible
+ * for loading the data into the database in the most efficient manner.
+ * We use queues and thread pools to ensure multiple threads are loading
+ * concurrently in large batches.
+ *
+ * @author Akara Sucharitakul
+ */
+public class Loader {
+
+    /** The batch size of a single batch. */
+    public static final int BATCHSIZE = 1000;
+
+    /** The recycling pool size is 3 times the size of the batch. */
+    public static final int POOLSIZE = 3 * BATCHSIZE;
+
+    /** The number of errors before exiting. */
+    public static final int ERROR_THRESHOLD = 50;
+
+    public static final int LOAD_THREADS = 5;
+
+    private static Logger logger =
+            Logger.getLogger(Loader.class.getName());
+
+    private static AtomicInteger errorCount = new AtomicInteger();
+
+    private static ConcurrentHashMap<String, Loader> typeMap =
+                            new ConcurrentHashMap<String, Loader>();
+
+    private static ConcurrentHashMap<Class, LoadablePool> poolMap =
+                            new ConcurrentHashMap<Class, LoadablePool>();
+
+    private static ArrayList<Thread> mainLoaders = new ArrayList<Thread>();
+
+    // This is a single processing pool for processing data preps.
+    private static ExecutorService processor =
+                                Executors.newCachedThreadPool();
+
+    private String name;
+    AtomicInteger loadCount;
+
+    LoadablePool<? extends Loadable> loadablePool;
+
+    // A Loadable type database loading pool.
+    ExecutorService pool;
+    ConcurrentLinkedQueue<Loadable> queue;
+
+    /**
+     * Obtains the instance of the loader for a given loadable type.
+     * @param clazz The loadable type
+     * @return The loader for this type name, or a new loader if none exists
+     */
+    static Loader getInstance(Class<? extends Loadable> clazz) {
+        // We may need to change this to a configurable thread pool size
+        // on a per-type basis. This is the only place to change.
+
+        String name = clazz.getName();
+        Loader loader = new Loader();
+        Loader oldEntry = typeMap.putIfAbsent(name, loader);
+
+        if (oldEntry != null)
+            loader = oldEntry;
+
+        loader.validate(name);
+        return loader;
+    }
+
+    private synchronized void validate(String name) {
+        if (this.name == null)
+            this.name = name;
+        if (loadCount == null)
+            loadCount = new AtomicInteger(0);
+        if (queue == null)
+            queue = new ConcurrentLinkedQueue<Loadable>();
+
+        // We may need to change this to a configurable thread pool size
+        // on a per-type basis. This is the only place to change.
+        if (pool == null)
+            pool = Executors.newFixedThreadPool(LOAD_THREADS);
+            // pool = Executors.newCachedThreadPool();
+    }
+
+    private static <T extends Loadable> LoadablePool<T>
+            getLoadablePool(Class<T> clazz) {
+        LoadablePool<T> pool = new LoadablePool<T>(3 * BATCHSIZE, clazz);
+        @SuppressWarnings("unchecked")
+                LoadablePool<T> oldEntry = poolMap.putIfAbsent(clazz, pool);
+
+        if (oldEntry != null) {
+            pool = oldEntry;
+        }
+
+        return pool;
+    }
+
+    /**
+     * Sets the URL for the connection to the database.
+     * @param url The connection URL
+     */
+    public static void setConnectionURL(String url) {
+        ThreadConnection.connectionURL = url;
+    }
+
+    public static void setJDBCDriverClassName(String driver)
+            throws ClassNotFoundException, IllegalAccessException,
+            InstantiationException {
+        // Just load the DB driver class.
+        Class.forName(driver).newInstance();
+    }
+
+    /**
+     * Uses the loadable to clear the database through the loadable's
+     * clear statement.
+     * @param clazz The loadable class to use
+     */
+    public static void clear(Class<? extends Loadable> clazz) {
+        Loadable loadable = null;
+        try {
+            loadable = clazz.newInstance();
+        } catch (Exception ex) {
+            logger.log(Level.SEVERE, "Error instantiating loader class.", ex);
+            increaseErrorCount();
+        }
+
+        if (loadable != null) {
+            final Loadable l = loadable;
+        Future f = l.loader.pool.submit(new Runnable() {
+
+            public void run() {
+                ThreadConnection c = ThreadConnection.getInstance();
+                try {
+                    c.prepareStatement(l.getClearStatement());
+                    c.executeUpdate();
+                } catch (SQLException e) {
+                    logger.log(Level.SEVERE, l.loader.name + ": " +
+                            e.getMessage(), e);
+                    increaseErrorCount();
+                }
+            }
+        });
+        while (!f.isDone() || f.isCancelled()) {
+            try {
+                Thread.sleep(200);
+            } catch (InterruptedException e) {
+                    logger.log(Level.WARNING, l.loader.name + ": Interrupted " +
+                            "while waiting to clear table.", e);
+            }
+        }
+    }
+    }
+
+    /**
+     * Loads the loadable into the database. Note that the loading is done
+     * asynchronously and is divided into two phases: 1) The preparation
+     * phase where all field values are generated and 2) Loading phase. These
+     * may be performed by different threads. The waitProcessing method
+     * will gracefully shut down the processing infrastructure and wait until
+     * all preparation is done. Shutdown will wait until all data loading
+     * is done.
+     * @param clazz The loadable class
+     * @param occurrences The number of load iterations
+     */
+    public static void load(Class<? extends Loadable> clazz, int occurrences) {
+
+        final Class<? extends Loadable> c = clazz;
+        final int occ = occurrences;
+        Thread mainLoader = new Thread() {
+
+            @Override
+            public void run() {
+                for (int i = 0; i < occ; i++) {
+                    Loadable loadable = null;
+                    try {
+                        loadable = getLoadablePool(c).getLoadable();
+                    } catch (Exception ex) {
+                        logger.log(Level.SEVERE, "Error obtaining loadable", ex);
+                        increaseErrorCount();
+                    }
+                    if (loadable != null) {
+                        final Loadable l = loadable;
+        processor.execute(new Runnable() {
+
+            public void run() {
+                try {
+                    l.prepare();
+                    l.loader.add(l);
+                } catch (Exception e) {
+                    logger.log(Level.WARNING, e.getMessage(), e);
+                    Loader.increaseErrorCount();
+                }
+            }
+        });
+    }
+                }
+            }
+        };
+        mainLoaders.add(mainLoader);
+        mainLoader.start();
+    }
+
+    public static void exec(Runnable r) {
+        processor.execute(r);
+    }
+
+    /**
+     * Execute the post loads provided by the loadable.
+     * @param clazz The loadable class
+     */
+    public static void postLoad(Class<? extends Loadable> clazz) {
+        Loadable loadable = null;
+        try {
+            loadable = clazz.newInstance();
+        } catch (Exception ex) {
+            logger.log(Level.SEVERE, "Error instantiating loader class.", ex);
+            increaseErrorCount();
+        }
+
+        if (loadable != null) {
+
+            final Loadable l = loadable;
+        l.loader.pool.submit(new Runnable() {
+
+            public void run() {
+                try {
+                    l.postLoad();
+                } catch (Exception e) {
+                    logger.log(Level.WARNING, l.loader.name + ": " +
+                                                    e.getMessage(), e);
+                    increaseErrorCount();
+                }
+            }
+        });
+    }
+    }
+
+
+    private void add(Loadable l) {
+        queue.add(l);
+        int c = loadCount.incrementAndGet();
+        if (c % BATCHSIZE == 0)
+            flush(c);
+    }
+
+    private void flush(final int batchCount) {
+        pool.submit(new Runnable() {
+            public void run() {
+                ThreadConnection c = ThreadConnection.getInstance();
+                c.processBatch(name, batchCount, queue);
+            }
+        });
+    }
+
+    /**
+     * Terminates the preparation infrastructure and waits until all data
+     * preparation is done.
+     */
+    public static void waitProcessing() {
+        // Wait for the main loaders
+        for (Thread mainLoader : mainLoaders) {
+            for (;;)
+                try {
+                    mainLoader.join();
+                    break;
+                } catch (InterruptedException e) {
+                    logger.log(Level.WARNING, e.getMessage(), e);
+                }
+        }
+        // We ensure the process pool is cleared, first.
+        if (processor != null) {
+        processor.shutdown();
+        boolean terminated = false;
+        while (!terminated)
+            try {
+                terminated = processor.awaitTermination(1, TimeUnit.HOURS);
+            } catch (InterruptedException e) {
+            }
+        processor = null;
+        }
+    }
+
+    /**
+     * Terminates the preparation infrastructure (if still alive) and
+     * then the loading infrastructure. Will return only after all the
+     * loadables in the queue are loaded.
+     */
+    public static void shutdown() {
+        waitProcessing();
+        for (Loader entry : typeMap.values())
+            entry.flush(0);
+        for (Loader entry : typeMap.values())
+            entry.pool.shutdown();
+        for (Loader entry : typeMap.values()) {
+            while (!entry.pool.isTerminated())
+                try {
+                    entry.pool.awaitTermination(1, TimeUnit.HOURS);
+                } catch (InterruptedException e) {
+                }
+        }
+        typeMap.clear();
+        ThreadConnection.closeConnections();
+    }
+
+    /**
+     * Increments the global error count. If the count is beyond the threshold,
+     * the loader will terminate.
+     */
+    public static void increaseErrorCount() {
+        if (errorCount.incrementAndGet() > ERROR_THRESHOLD)
+            logger.severe("Error count exceeded threshold of " +
+                    ERROR_THRESHOLD + "! Exiting.");
+            System.err.println("Error count exceeded threshold of " +
+                    ERROR_THRESHOLD + "! Exiting.");
+            System.exit(2);
+    }
+}

Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadConnection.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadConnection.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadConnection.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadConnection.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id: ThreadConnection.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader.framework;
+
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Queue;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Wraps a java.sql.connection on a per-thread basis.
+ */
+public class ThreadConnection {
+
+    private static Logger logger =
+            Logger.getLogger(ThreadConnection.class.getName());
+
+    private static ThreadLocal<ThreadConnection> resource =
+            new ThreadLocal<ThreadConnection>() {
+        public ThreadConnection initialValue() {
+            return new ThreadConnection();
+        }
+    };
+
+    private static boolean COMMIT_TX = Boolean.parseBoolean(
+                                    System.getProperty("commit.tx", "true"));
+    private static final List<ThreadConnection> CONNECTIONLIST =
+            Collections.synchronizedList(new ArrayList<ThreadConnection>());
+
+    public static String connectionURL;
+
+    private Connection conn;
+    private String statementText;
+    private PreparedStatement statement;
+    private int currentBatch;
+    private boolean closed = false;
+
+    /**
+     * The batch buffer buffers the loadables to be added in a batch.
+     * These have to be per-thread and the only reason they are
+     * maintained by the ThreadConnection. Otherwise we need to keep
+     * allocating and collecting. And since we already have the threadlocal,
+     * there is no more overhead getting to it.
+     */
+    Loadable[] batchBuffer;
+
+    private ThreadConnection() {
+        CONNECTIONLIST.add(this);
+    }
+
+    public static ThreadConnection getInstance() {
+        return resource.get();
+    }
+
+    boolean ensureConnection() {
+        if (closed) {
+            logger.severe("Connection used after closure!");
+            Loader.increaseErrorCount();
+            return false;
+        }
+
+        try {
+            if (conn == null || conn.isClosed()) {
+                conn = DriverManager.getConnection(connectionURL);
+                statement = null;
+                statementText = null;
+            }
+        } catch (SQLException e) {
+            logger.log(Level.SEVERE, "Error connecting to DB", e);
+            Loader.increaseErrorCount();
+            return false;
+        }
+        return true;
+    }
+
+    boolean resetConnection() {
+        if (closed) {
+            logger.severe("Connection used after closure!");
+            Loader.increaseErrorCount();
+            return false;
+        }
+
+        try {
+            conn = DriverManager.getConnection(connectionURL);
+            statement = null;
+            statementText = null;
+        } catch (SQLException e) {
+            logger.log(Level.SEVERE, "Error connecting to DB", e);
+            Loader.increaseErrorCount();
+            return false;
+        }
+        return true;
+    }
+
+    public PreparedStatement prepareStatement(String statementText)
+            throws SQLException {
+        if (conn == null)
+            ensureConnection();
+        if (!statementText.equals(this.statementText)) {
+            this.statementText = statementText;
+            statement = conn.prepareStatement(statementText);
+        }
+        return statement;
+    }
+
+    public void addBatch() throws SQLException {
+        statement.addBatch();
+        ++currentBatch;
+    }
+
+    public void executeUpdate() throws SQLException {
+        if (ensureConnection()) {
+            statement.executeUpdate();
+            if (COMMIT_TX)
+                conn.commit();
+        }
+    }
+
+    void processBatch(String name, int batchCount,
+            Queue<? extends Loadable> queue) {
+        // First we need to save the load objects from the queue
+        // so we do not loose them in case we need to retry.
+        if (batchBuffer == null) {
+            batchBuffer = new Loadable[Loader.BATCHSIZE];
+        }
+        int count = 0;
+        for (; count < Loader.BATCHSIZE; count++) {
+            Loadable l = queue.poll();
+            if (l == null)
+                break;
+            batchBuffer[count] = l;
+        }
+
+        if (count == 0) // Nothing to load.
+            return;
+
+        // Then we load our objects into the DB, retrying the whole
+        // saved ones in case we run into a closed connection.
+        if (!ensureConnection())
+            return;
+
+        String batchName;
+        if (batchCount > 0)
+            batchName = "object batch " + (batchCount - count + 1) + " - " +
+                                batchCount + '.';
+        else
+            batchName = "final " + count +  " object batch.";
+
+        int flushed = 0;
+        for (int retry = 0; retry < 2; retry++) {
+            try {
+                for (int i = flushed; i < count; i++) {
+                    batchBuffer[i].load();
+
+                    // Each Loadable object may load more than 1 record.
+                    // So we need to check for the number of records
+                    // in the batch. If it is more than batchsize, we
+                    // need to flush the records, too.
+                    if (currentBatch >= Loader.BATCHSIZE) {
+                        flush();
+                        flushed += currentBatch;
+                        currentBatch = 0;
+                        logger.finer(name + ": Flushed " + flushed +
+                                " records in " + batchName);
+                    }
+                }
+                if (currentBatch > 0) {
+                    flush();
+                    flushed += currentBatch;
+                    currentBatch = 0;
+                    logger.finer(name + ": Flushed final " + flushed +
+                            " records in " + batchName);
+                }
+                logger.fine(name + ": Loaded " + batchName);
+                break; // We won't retry if everything is OK.
+            } catch (BatchUpdateException e) {
+                if (retry == 0) {
+                    resetConnection();
+                    logger.log(Level.WARNING, name +
+                                                ": Retry loading.", e);
+                } else {
+                    int[] stats = e.getUpdateCounts();
+                    int successes = 0;
+                    for (int stat : stats) {
+                        if (stat != Statement.EXECUTE_FAILED)
+                            ++successes;
+                    }
+                    if (successes == 0) {
+                        logger.log(Level.WARNING, name +
+                                ": Failed to update.", e);
+                        Loader.increaseErrorCount();
+                    }
+                }
+            } catch (SQLException e) {
+                if (retry == 0) {
+                    resetConnection();
+                    logger.log(Level.WARNING, name + ": Retry loading.",
+                                                                    e);
+                } else {
+                    logger.log(Level.WARNING, e.getMessage(), e);
+                    Loader.increaseErrorCount();
+                }
+            }
+        }
+
+        // Once we're done with this buffer, don't hold on to the objects.
+        // Return them to the pool so we don't bloat memory.
+        for (int i = 0; i < batchBuffer.length; i++) {
+            batchBuffer[i].pool.putLoader(batchBuffer[i]);
+            batchBuffer[i] = null;
+    }
+    }
+
+    void flush() throws SQLException {
+        statement.executeBatch();
+        if (COMMIT_TX)
+            conn.commit();
+    }
+
+
+    void close() throws SQLException {
+        closed = true;
+        if (statement != null)
+            statement.close();
+        if (conn != null)
+            conn.close();
+    }
+
+    static void closeConnections() {
+        synchronized (CONNECTIONLIST) {
+            for (ThreadConnection c : CONNECTIONLIST)
+                try {
+                    c.close();
+                } catch (SQLException e) {
+                    logger.log(Level.SEVERE, e.getMessage(), e);
+                    Loader.increaseErrorCount();
+                }
+            CONNECTIONLIST.clear();
+        }
+    }
+}

Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadResource.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadResource.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadResource.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/loader/framework/ThreadResource.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id: ThreadResource.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.loader.framework;
+
+import com.sun.faban.driver.util.Random;
+
+import java.util.logging.Logger;
+import java.text.SimpleDateFormat;
+import java.text.DateFormat;
+
+public class ThreadResource {
+
+    Logger logger = Logger.getLogger(ThreadResource.class.getName());
+
+    private static ThreadLocal<ThreadResource> resource =
+            new ThreadLocal<ThreadResource>() {
+        public ThreadResource initialValue() {
+            return new ThreadResource();
+        }
+    };
+
+    Random random;
+    StringBuilder buffer;
+    SimpleDateFormat dateFormat;
+
+    private ThreadResource() {
+        buffer = new StringBuilder(256);
+        random = new Random();
+    }
+
+    public StringBuilder getBuffer() {
+        buffer.setLength(0); // Make sure we clear it
+        return buffer;
+    }
+
+    public Random getRandom() {
+        return random;
+    }
+
+    /**
+     * DateFormat is not thread safe. We need to include it into the
+     * ThreadResource.
+     * @return The thread instance of DateFormat.
+     */
+    public DateFormat getDateFormat() {
+        if (dateFormat == null)
+            dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+        return dateFormat;
+    }
+
+    public static ThreadResource getInstance() {
+        return resource.get();
+    }
+}

Propchange: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/
------------------------------------------------------------------------------
    svn:mergeinfo = 

Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/GrowthTest.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/GrowthTest.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/GrowthTest.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/GrowthTest.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id: GrowthTest.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.util;
+
+/**
+ * Code to test the growth function and match it to existing research.
+ */
+public class GrowthTest {
+
+    // We use on average of 3.5 tags per event. Random 1..6 tags.
+    // Once we know the tag count, we have to select tags.
+
+    /*
+
+    http://tagsonomy.com/index.php/dynamic-growth-of-tag-clouds/
+
+    As of this writing, a little over 700 users have tagged it, with 450+
+    unique tags, roughly two-thirds of which tags were (of course) used by
+    one and only one user.
+
+    It took only 10 users (not quite 1.5% of the current total) before the
+    top 3 tags were tagging ontology folksonomy, conveying much the same
+    sense, with only the use of tagging instead of tags making this
+    different from the current set of 3.
+    */
+
+
+    public static double cumuLogistic(
+            double x, double mean, double scale) {
+        return 0.5d + Math.tanh((x - mean) / (2 * scale)) / 2d;
+    }
+
+    public static double cumuHalfLogistic(double x, double scale) {
+        return (1d - Math.pow(Math.E, -x/scale)) / (1d + Math.pow(Math.E, -x/scale));
+    }
+
+    public static double sigmoid(double x, double mean, double scale) {
+        return 1d / (1d + Math.pow(Math.E, -((x / scale) - mean)));
+    }
+
+    public static void main(String[] args) {
+
+        int limit = 5000;
+        int mean = 5000;
+        int scale = 500;
+        for (int x = 0; x < 10000; x += 100) {
+            int y = (int) Math.round(limit * cumuLogistic(x, 5000, 1000));
+            int y2 = (int) Math.round(limit * cumuHalfLogistic(x, 10000)); // Done
+            int y3 = (int) Math.round(limit * sigmoid(x, 6, 1000));
+            System.out.println("-> " + x + ',' + y + ',' + y2 + ',' + y3);
+        }
+    }
+
+}

Added: incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/MemCacheUtility.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/MemCacheUtility.java?rev=745184&view=auto
==============================================================================
--- incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/MemCacheUtility.java (added)
+++ incubator/olio/workload/php/trunk/src/org/apache/olio/workload/util/MemCacheUtility.java Tue Feb 17 18:25:43 2009
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * $Id: MemCacheUtility.java,v 1.1.1.1 2008/09/29 22:33:08 sp208304 Exp $
+ */
+package org.apache.olio.workload.util;
+
+import com.danga.MemCached.MemCachedClient;
+import com.danga.MemCached.SockIOPool;
+import com.sun.faban.common.NameValuePair;
+import com.sun.faban.common.TextTable;
+
+import java.util.*;
+import java.util.logging.Logger;
+
+
+
+/**
+ *
+ * @author Kim LiChong
+ */
+public class MemCacheUtility {
+
+    private static MemCachedClient cache = null;
+    private String[] serverList = null;
+    static Logger logger = Logger.getLogger(
+            MemCacheUtility.class.getName());
+    private long baseTime = Long.MIN_VALUE;
+
+
+    /** This constructor creates a new instance of MemCacheUtility
+     A memcache client is created with a pool of servers.
+     * 
+     * @param servers ArrayList of NameValuePair<Integer> of servers:port numbers.
+     */
+    public MemCacheUtility(ArrayList<NameValuePair<Integer>> servers) {
+        if (cache == null) {
+            // the env memcachedInstances is in the
+            // form host1:port1, host2:port2, etc.
+            // in an ArrayList 
+            //String servers = locator.getString("memcachedInstances");
+
+
+            serverList = new String[servers.size()];
+
+            serverList = convertNameValueToStringArray(servers);
+
+            //logger.info("size of the array is " + serverList.length);
+            //String[] serverList = servers.split(",?[ *]");
+            SockIOPool pool = SockIOPool.getInstance("livePool");
+            pool.setServers(serverList);
+            pool.initialize();
+
+            cache = new MemCachedClient();
+            cache.setPoolName("livePool");
+        }               
+    }
+
+    /** This constructor creates a new instance of MemCacheUtility
+     A memcache client is created with a pool of servers.
+     * 
+     * @param servers String []  servers:port.
+     */
+    public MemCacheUtility(String[] servers) {
+        if (cache == null) {
+            SockIOPool pool = SockIOPool.getInstance("livePool");
+            pool.setServers(servers);
+            pool.initialize();
+
+            cache = new MemCachedClient();
+            cache.setPoolName("livePool");
+        }
+    }
+
+    /*
+    * This method is a convenience method to convert ArrayList<NameValuePair<Integer> to
+    * a String array of server:port server2:port.
+    * @param servers  ArrayList<NameValuePair<Integer>>
+    * @return String []
+    */
+
+    public static String[] convertNameValueToStringArray (
+            ArrayList<NameValuePair<Integer>> servers) {
+        String [] serverArr = new String[servers.size()];
+        int index = 0;
+        for (NameValuePair<Integer> serverEntry : servers) {
+            serverArr[index++] = serverEntry.name + ":" + serverEntry.value;
+        }
+        return serverArr;
+    }
+
+    /** Retrieves memcache stats for each instance of MemCacheUtility.
+     *  A TextTable will be produced for each server used to create an
+     *  instance of MemCacheUtility.  Used to persist stats file for each server.
+     *  Returning Map whose key is the servername, the value is a TextTable of statistics
+     *  @return Map 
+     */
+
+    public Map<String, TextTable> getStats() {
+
+
+        Map memcacheStats = cache.stats();
+        //cache.stats() will return a Map whose key  is the name of the memcache server
+        //and whose value is a Map with the memcache statistics
+
+
+        //logger.info("Map size returning is " + memcacheStats.size());
+
+        //produce a TextTable for each server listed
+        Map<String, TextTable> returnMap = new HashMap<String, TextTable>();
+
+        TextTable outputTextTable = null;
+
+        Set<Map.Entry> statEntries = memcacheStats.entrySet();
+
+        //set counter to allow to set number of columns to output
+        for (Map.Entry statEntry : statEntries) {
+            String key = (String) statEntry.getKey();
+            Map statsMap = (Map) statEntry.getValue();
+            //is this case, it is a Map with the statistics
+            //get size so we know how big to make TextTable
+            outputTextTable = new TextTable(statsMap.size(), 2);
+            //set Header
+            outputTextTable.setHeader(0, "Parameter");
+            outputTextTable.setHeader(1, "Value");
+            //outputTextTable.setHeader(2, "for " + key);
+            //get this value's iterator
+            Set<Map.Entry> statsMapEntries = statsMap.entrySet();
+            int counter=0;
+            for (Map.Entry statsMapEntry : statsMapEntries) {
+                outputTextTable.setField(counter, 0,
+                        (CharSequence) statsMapEntry.getKey());
+                outputTextTable.setField(counter++, 1,
+                        (CharSequence) statsMapEntry.getValue());
+            }
+            //add each TextTable for each server listed to return Map.
+            returnMap.put(key, outputTextTable);
+
+        }
+        return returnMap;
+    }
+
+    /* This method is used for dynamic memcache stats gathering.
+    * The TextTable will contain all memcache server instances in columns
+    * and the server parameters in rows
+    * @return TextTable
+    *@see com.sun.faban.common.TextTable
+    */
+
+    public TextTable getTemporaryStats() {
+        Long time = System.currentTimeMillis();
+        int elapsed = 0;
+        if (baseTime == Long.MIN_VALUE)
+            baseTime = time;
+        else
+            elapsed = (int) (time - baseTime);
+
+        String elapsedSecs = String.format("%.3f", elapsed/1000d);
+
+        Map memcacheStats = cache.stats();
+        //cache.stats() will return a Map whose key  is the name of the memcache server
+        //and whose value is a Map with the memcache statistics
+        TextTable outputTextTable = null;
+        Set<Map.Entry> serverEntries = memcacheStats.entrySet();
+
+        //set counter to allow to set number of columns to output
+        int counter = 0;
+        int columnIndex = 0;
+
+        //reset the iterator
+        for (Map.Entry serverEntry : serverEntries) {
+            String key = (String) serverEntry.getKey();
+            Map statsMap = (Map) serverEntry.getValue();
+            if (outputTextTable == null) {
+                // One extra row for elapsed time, one extra header column.
+                outputTextTable = new TextTable(statsMap.size(),
+                        serverEntries.size() + 2);
+            }
+            //is this case, it is a Map with the statistics
+            //get size so we know how big to make TextTable
+            // the number of rows is the number of stats
+            // the number of columns is how many server instances there are
+            //set Header
+            outputTextTable.setHeader(0, "Elapsed (sec)");
+            outputTextTable.setHeader(1, "Parameter");
+            outputTextTable.setHeader(columnIndex + 2, key);
+
+            //get this value's iterator
+            Set<Map.Entry> statsMapEntries = statsMap.entrySet();
+            counter=0; //reset counter
+
+            // Populate the rest of the table.
+            for (Map.Entry statsMapEntry : statsMapEntries) {
+                outputTextTable.setField(counter, 0, elapsedSecs);
+                outputTextTable.setField(counter, 1,
+                        (CharSequence) statsMapEntry.getKey());
+                outputTextTable.setField(counter++, columnIndex + 2,
+                        (CharSequence) statsMapEntry.getValue());
+            }
+            ++columnIndex;
+        }
+        return outputTextTable;
+    }
+
+    /*
+    This main method is used to gather dynamic statistics on memcache server instances.
+    *  It expects at least 4 arguments:
+    *
+    *  host:server host:server (additional server instances can be designated as host1:port1 host1:port2 OR host2:port etc.
+    * -s start time:  the ramp up time, in seconds.  (status collection does not take place during the ramp up)
+    * -e end time: the steady state, in seconds. (time to do the statistics data collection)
+    * -i interval time: the snapshot period to collect the stats, in seconds.
+    *
+    *     Usage:  java com.sun.web20.MemCacheUtility server:port [server2:port server3:port] -s startTime -e endTime -i interval
+    *     eg. java com.sun.web20.util.MemCacheUtility server1:12100 server2:12100 -s 300 -e 600 -i 3
+    *     This will sleep for 300 seconds during ramp up, collect for 600 seconds with an interval of 3 seconds between
+    *     each snapshot.
+    *     @param args String []
+    *
+    */
+    public static void main (String[] args) {
+
+        if (args==null || args.length < 4) {//minimum amount of args - one server, -s, -e, -i
+            System.out.println("Usage:  java com.sun.web20.MemCacheUtility server:port [server2:port server3:port] -s startTime -e endTime -i interval");
+            System.out.println(" where startTime = ramp up time in seconds.  Statistics collection will NOT occur during ramp up time and will sleep for startTime period");
+            System.out.println(" endTime = steady State time in seconds.  Statistics collection will only occur during the steady state period");
+            System.out.println(" interval = time between statistics collection snapshots, in seconds.");
+        }
+
+
+        int startTime = 0;
+        int endTime = 0;
+        int intervalTime = 0;
+        LinkedHashSet<String> serverSet = new LinkedHashSet<String>();
+        for (int i = 0; i < args.length; i++) {
+            if (args[i].startsWith("-s")) {
+                if (args[i].length() > 2) // -sarg
+                    startTime = Integer.parseInt(args[i].substring(2)) * 1000;
+                else // -s arg
+                    startTime = Integer.parseInt(args[++i]) * 1000;
+            } else if (args[i].startsWith("-e")) {
+                if (args[i].length() > 2) // -earg
+                    endTime = Integer.parseInt(args[i].substring(2)) * 1000;
+                else // -e arg
+                    endTime = Integer.parseInt(args[++i]) * 1000;
+            } else if (args[i].startsWith("-i")) {
+                if (args[i].length() > 2) // -iarg
+                    intervalTime =
+                            Integer.parseInt(args[i].substring(2)) * 1000;
+                else // -i arg
+                    intervalTime = Integer.parseInt(args[++i])* 1000;
+            } else if (args[i].contains(":")) {// host:port pair
+                serverSet.add(args[i]);
+            } else { // host only. Append default port 11211.
+                serverSet.add(args[i] + ":11211");
+            }
+        }
+
+        //finished processing all of the args.  populate server list
+        String memCacheServers[] = new String[serverSet.size()];
+        memCacheServers = serverSet.toArray(memCacheServers);
+
+        logger.info("Starting memcache stats");
+
+        //collect only during steady state
+        MemCacheUtility memCacheUtil = new MemCacheUtility(memCacheServers);
+
+        try {
+            Timer timer = new Timer();
+            MemCacheTask task = new MemCacheTask(memCacheUtil);
+            timer.scheduleAtFixedRate(task, startTime, intervalTime);
+            //only print stats for steady state period
+            Thread.sleep(endTime);
+            //wake up and stop printing stats
+            timer.cancel();
+        } catch (InterruptedException ex)  {
+            ex.printStackTrace();
+            return;
+        }
+    }
+
+    /* class for TimerTask */
+
+    private static class MemCacheTask extends TimerTask {
+
+        private MemCacheUtility memCacheUtility;
+
+        public MemCacheTask(MemCacheUtility memCacheUtil) {
+            memCacheUtility = memCacheUtil;
+
+        }
+
+        public void run() {
+
+            System.out.println(memCacheUtility.getTemporaryStats());
+
+        }
+
+    }
+
+
+
+
+}



Mime
View raw message