incubator-olio-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject svn commit: r831988 - in /incubator/olio/workload/java/trunk: bin/ src/org/apache/olio/workload/driver/ src/org/apache/olio/workload/fsloader/ src/org/apache/olio/workload/loader/
Date Mon, 02 Nov 2009 18:08:42 GMT
Author: shanti
Date: Mon Nov  2 18:08:41 2009
New Revision: 831988

URL: http://svn.apache.org/viewvc?rev=831988&view=rev
Log:
Fix for OLIO-40. Schema and loader changes to add userid as 
primary key for PERSON, and use userid instead of username in 
PERSON_PERSON.

Modified:
    incubator/olio/workload/java/trunk/bin/schema.sql
    incubator/olio/workload/java/trunk/src/org/apache/olio/workload/driver/UIDriver.java
    incubator/olio/workload/java/trunk/src/org/apache/olio/workload/fsloader/FileLoader.java
    incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Address.java
    incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Friends.java
    incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/LoadController.java
    incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Person.java

Modified: incubator/olio/workload/java/trunk/bin/schema.sql
URL: http://svn.apache.org/viewvc/incubator/olio/workload/java/trunk/bin/schema.sql?rev=831988&r1=831987&r2=831988&view=diff
==============================================================================
--- incubator/olio/workload/java/trunk/bin/schema.sql (original)
+++ incubator/olio/workload/java/trunk/bin/schema.sql Mon Nov  2 18:08:41 2009
@@ -56,6 +56,68 @@
 SET character_set_client = @saved_cs_client;
 
 --
+-- Table structure for table `PERSON`
+--
+
+DROP TABLE IF EXISTS `PERSON`;
+SET @saved_cs_client     = @@character_set_client;
+SET character_set_client = utf8;
+CREATE TABLE `PERSON` (
+  `USERID` INTEGER NOT NULL AUTO_INCREMENT,
+  `USERNAME` varchar(25) NOT NULL,
+  `PASSWORD` varchar(25) DEFAULT NULL,
+  `SUMMARY` text,
+  `TELEPHONE` varchar(25) DEFAULT NULL,
+  `IMAGETHUMBURL` varchar(100) DEFAULT NULL,
+  `IMAGEURL` varchar(100) DEFAULT NULL,
+  `FIRSTNAME` varchar(25) DEFAULT NULL,
+  `LASTNAME` varchar(25) DEFAULT NULL,
+  `EMAIL` varchar(90) DEFAULT NULL,
+  `TIMEZONE` varchar(25) DEFAULT NULL,
+  `ADDRESS_ADDRESSID` INTEGER DEFAULT NULL,
+  PRIMARY KEY (`USERID`),
+  KEY `FK_PERSON_ADDRESS_ADDRESSID` (`ADDRESS_ADDRESSID`),
+  CONSTRAINT `FK_PERSON_ADDRESS_ADDRESSID` FOREIGN KEY (`ADDRESS_ADDRESSID`) REFERENCES `ADDRESS`
(`ADDRESSID`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+SET character_set_client = @saved_cs_client;
+
+CREATE UNIQUE INDEX PERSON_USER_IDX  on PERSON (username);
+
+--
+-- Table structure for table `SOCIALEVENT`
+--
+
+DROP TABLE IF EXISTS `SOCIALEVENT`;
+SET @saved_cs_client     = @@character_set_client;
+SET character_set_client = utf8;
+CREATE TABLE `SOCIALEVENT` (
+  `SOCIALEVENTID` INTEGER NOT NULL AUTO_INCREMENT,
+  `DESCRIPTION` VARCHAR(500) NOT NULL,
+  `TITLE` varchar(100) DEFAULT NULL,
+  `SUBMITTERUSERNAME` varchar(25) DEFAULT NULL,
+  `SUMMARY` varchar(100) DEFAULT NULL,
+  `TELEPHONE` varchar(20) DEFAULT NULL,
+  `IMAGETHUMBURL` varchar(100) DEFAULT NULL,
+  `IMAGEURL` varchar(100) DEFAULT NULL,
+  `LITERATUREURL` varchar(100) DEFAULT NULL,
+  `EVENTTIMESTAMP` datetime DEFAULT NULL,
+  `TOTALSCORE` INTEGER DEFAULT NULL,
+  `NUMBEROFVOTES` INTEGER  DEFAULT NULL,
+  `DISABLED` INTEGER DEFAULT NULL,
+  `CREATEDTIMESTAMP` datetime DEFAULT NULL,
+  `ADDRESS_ADDRESSID` INTEGER DEFAULT NULL,
+  PRIMARY KEY (`SOCIALEVENTID`),
+  KEY `FK_SOCIALEVENT_ADDRESS_ADDRESSID` (`ADDRESS_ADDRESSID`),
+  CONSTRAINT `FK_SOCIALEVENT_ADDRESS_ADDRESSID` FOREIGN KEY (`ADDRESS_ADDRESSID`) REFERENCES
`ADDRESS` (`ADDRESSID`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+SET character_set_client = @saved_cs_client;
+ 
+-- CREATE INDEX SOCIALEVENT_DATE_IDX on SOCIALEVENT (EVENTTIMESTAMP);
+CREATE INDEX SOCIALEVENT_EVENTTIMESTAMP_IDX on SOCIALEVENT (eventtimestamp);
+CREATE INDEX SOCIALEVENT_CREATEDTIMESTAMP_IDX on SOCIALEVENT (createdtimestamp);
+CREATE INDEX SOCIALEVENT_SUBMITTERUSERNAME_IDX  on SOCIALEVENT (submitterUserName);
+
+--
 -- Table structure for table `COMMENTS_RATING`
 --
 
@@ -67,7 +129,7 @@
   `CREATIONTIME` datetime DEFAULT NULL,
   `COMMENTS` text,
   `RATING` INTEGER DEFAULT NULL,
-  `USERNAME_USERNAME` varchar(55) DEFAULT NULL,
+  `USERNAME_USERNAME` varchar(25) DEFAULT NULL,
   `SOCIALEVENT_SOCIALEVENTID` INTEGER DEFAULT NULL,
   PRIMARY KEY (`commentsid`),
   KEY `FK_COMMENTS_RATING_SOCIALEVENT_SOCIALEVENTID` (`SOCIALEVENT_SOCIALEVENTID`),
@@ -117,31 +179,6 @@
 SET character_set_client = @saved_cs_client;
 
 
---
--- Table structure for table `PERSON`
---
-
-DROP TABLE IF EXISTS `PERSON`;
-SET @saved_cs_client     = @@character_set_client;
-SET character_set_client = utf8;
-CREATE TABLE `PERSON` (
-  `USERNAME` varchar(25) NOT NULL,
-  `PASSWORD` varchar(25) DEFAULT NULL,
-  `SUMMARY` text,
-  `TELEPHONE` varchar(25) DEFAULT NULL,
-  `IMAGETHUMBURL` varchar(100) DEFAULT NULL,
-  `IMAGEURL` varchar(100) DEFAULT NULL,
-  `FIRSTNAME` varchar(25) DEFAULT NULL,
-  `LASTNAME` varchar(25) DEFAULT NULL,
-  `EMAIL` varchar(90) DEFAULT NULL,
-  `TIMEZONE` varchar(25) DEFAULT NULL,
-  `ADDRESS_ADDRESSID` INTEGER DEFAULT NULL,
-  PRIMARY KEY (`USERNAME`),
-  KEY `FK_PERSON_ADDRESS_ADDRESSID` (`ADDRESS_ADDRESSID`),
-  CONSTRAINT `FK_PERSON_ADDRESS_ADDRESSID` FOREIGN KEY (`ADDRESS_ADDRESSID`) REFERENCES `ADDRESS`
(`ADDRESSID`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
-SET character_set_client = @saved_cs_client;
-
 
 --
 -- Table structure for table `PERSON_PERSON`
@@ -151,12 +188,12 @@
 SET @saved_cs_client     = @@character_set_client;
 SET character_set_client = utf8;
 CREATE TABLE `PERSON_PERSON` (
-  `Person_USERNAME` varchar(25) NOT NULL,
-  `friends_USERNAME` varchar(25) NOT NULL,
-  PRIMARY KEY (`Person_USERNAME`,`friends_USERNAME`),
-  KEY `FK_PERSON_PERSON_friends_USERNAME` (`friends_USERNAME`),
-  CONSTRAINT `FK_PERSON_PERSON_Person_USERNAME` FOREIGN KEY (`Person_USERNAME`) REFERENCES
`PERSON` (`USERNAME`),
-  CONSTRAINT `FK_PERSON_PERSON_friends_USERNAME` FOREIGN KEY (`friends_USERNAME`) REFERENCES
`PERSON` (`USERNAME`)
+  `Person_USERID` INTEGER NOT NULL,
+  `friends_USERID` INTEGER NOT NULL,
+  PRIMARY KEY (`Person_USERID`,`friends_USERID`),
+  KEY `FK_PERSON_PERSON_friends_USERID` (`friends_USERID`),
+  CONSTRAINT `FK_PERSON_PERSON_Person_USERID` FOREIGN KEY (`Person_USERID`) REFERENCES `PERSON`
(`USERID`),
+  CONSTRAINT `FK_PERSON_PERSON_friends_USERID` FOREIGN KEY (`friends_USERID`) REFERENCES
`PERSON` (`USERID`)
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
 SET character_set_client = @saved_cs_client;
 
@@ -178,40 +215,6 @@
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
 SET character_set_client = @saved_cs_client;
 
---
--- Table structure for table `SOCIALEVENT`
---
-
-DROP TABLE IF EXISTS `SOCIALEVENT`;
-SET @saved_cs_client     = @@character_set_client;
-SET character_set_client = utf8;
-CREATE TABLE `SOCIALEVENT` (
-  `SOCIALEVENTID` INTEGER NOT NULL AUTO_INCREMENT,
-  `DESCRIPTION` VARCHAR(500) NOT NULL,
-  `TITLE` varchar(100) DEFAULT NULL,
-  `SUBMITTERUSERNAME` varchar(25) DEFAULT NULL,
-  `SUMMARY` varchar(100) DEFAULT NULL,
-  `TELEPHONE` varchar(20) DEFAULT NULL,
-  `IMAGETHUMBURL` varchar(100) DEFAULT NULL,
-  `IMAGEURL` varchar(100) DEFAULT NULL,
-  `LITERATUREURL` varchar(100) DEFAULT NULL,
-  `EVENTTIMESTAMP` datetime DEFAULT NULL,
-  `TOTALSCORE` INTEGER DEFAULT NULL,
-  `NUMBEROFVOTES` INTEGER  DEFAULT NULL,
-  `DISABLED` INTEGER DEFAULT NULL,
-  `CREATEDTIMESTAMP` datetime DEFAULT NULL,
-  `ADDRESS_ADDRESSID` INTEGER DEFAULT NULL,
-  PRIMARY KEY (`SOCIALEVENTID`),
-  KEY `FK_SOCIALEVENT_ADDRESS_ADDRESSID` (`ADDRESS_ADDRESSID`),
-  CONSTRAINT `FK_SOCIALEVENT_ADDRESS_ADDRESSID` FOREIGN KEY (`ADDRESS_ADDRESSID`) REFERENCES
`ADDRESS` (`ADDRESSID`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
-SET character_set_client = @saved_cs_client;
- 
--- CREATE INDEX SOCIALEVENT_DATE_IDX on SOCIALEVENT (EVENTTIMESTAMP);
-CREATE INDEX SOCIALEVENT_EVENTTIMESTAMP_IDX on SOCIALEVENT (eventtimestamp);
-CREATE INDEX SOCIALEVENT_CREATEDTIMESTAMP_IDX on SOCIALEVENT (createdtimestamp);
-CREATE INDEX SOCIALEVENT_SUBMITTERUSERNAME_IDX  on SOCIALEVENT (submitterUserName);
-
 
 --
 -- Table structure for table `SOCIALEVENTTAG`

Modified: incubator/olio/workload/java/trunk/src/org/apache/olio/workload/driver/UIDriver.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/java/trunk/src/org/apache/olio/workload/driver/UIDriver.java?rev=831988&r1=831987&r2=831988&view=diff
==============================================================================
--- incubator/olio/workload/java/trunk/src/org/apache/olio/workload/driver/UIDriver.java (original)
+++ incubator/olio/workload/java/trunk/src/org/apache/olio/workload/driver/UIDriver.java Mon
Nov  2 18:08:41 2009
@@ -591,6 +591,8 @@
         params.add(new StringPart("last_name", parameters[3]));
         params.add(new StringPart("email", parameters[4]));
         params.add(new StringPart("telephone",parameters[5]));
+        params.add(new StringPart("summary", parameters[6]));
+        params.add(new StringPart("timezone", parameters[7]));
         String[] addressArr = prepareAddress();
         params.add(new StringPart("street1", addressArr[0]));
         params.add(new StringPart("street2", addressArr[1]));
@@ -598,8 +600,6 @@
         params.add(new StringPart("state", addressArr[3]));
         params.add(new StringPart("zip", addressArr[4]));
         params.add(new StringPart("country", addressArr[5]));
-        params.add(new StringPart("summary", parameters[6]));
-        params.add(new StringPart("timezone", parameters[7]));
         params.add(new FilePart("upload_person_image", personImg));
         /****
         Part[] parts = new Part[params.size()];

Modified: incubator/olio/workload/java/trunk/src/org/apache/olio/workload/fsloader/FileLoader.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/java/trunk/src/org/apache/olio/workload/fsloader/FileLoader.java?rev=831988&r1=831987&r2=831988&view=diff
==============================================================================
--- incubator/olio/workload/java/trunk/src/org/apache/olio/workload/fsloader/FileLoader.java
(original)
+++ incubator/olio/workload/java/trunk/src/org/apache/olio/workload/fsloader/FileLoader.java
Mon Nov  2 18:08:41 2009
@@ -1,4 +1,4 @@
- /*
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -14,34 +14,65 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
+ *  $Id$
  */
 package org.apache.olio.workload.fsloader;
 
+import com.sun.faban.harness.util.FileHelper;
 import org.apache.olio.workload.util.ScaleFactors;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.channels.FileChannel;
 import java.util.Formatter;
 import java.util.ArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 
 public class FileLoader {
 
+    /**
+     * The default number of loader threads. Since the threads are blocking
+     * for I/O, this number may be quite high.
+     */
+    public static final int DEFAULT_LOADER_THREADS = 128;
+
+    /**
+     * The frequency to report status, in number of files.
+     */
+    public static final int REPORT_FREQUENCY = 10000;
+
+    private static final String DIR_PATTERN = File.separator + "%03d" + File.separator +
"%03d" + File.separator;
+
     private static Logger logger = Logger.getLogger(FileLoader.class.getName());
+    private static ExecutorService threadPool;
+    private static ArrayList<FileChannel> openFiles = new ArrayList<FileChannel>();
+    private static LoaderPool loaderPool;
 
     public static void main(String[] args) throws Exception {
-        System.out.println ("args =");
-        for (String arg: args) {
-            System.out.println (arg);
-        }
         String srcDir = args[0];
         String destDir = args[1];
         ScaleFactors.setActiveUsers(Integer.parseInt(args[2]));
+
+        int loaderThreads;
+        if (args.length > 3)
+            loaderThreads = Integer.parseInt(args[3]);
+        else
+            loaderThreads = DEFAULT_LOADER_THREADS;
+
+        // Throttle the job producer so that the objects in the work
+        // queue are limited to 10x the worker threads.
+        loaderPool = new LoaderPool(loaderThreads * 10);
+
         srcDir += File.separator;
 
         // Clear the dest dir
@@ -52,125 +83,147 @@
         }
 
         logger.info("Deleting files in " + destDir);
-        File[] list = dest.listFiles();
-        for (File f : list) {
-            if (!f.delete())
-                logger.warning("Error deleting file " + f.getName());
-        }
-
-        ArrayList<LoaderThread> loaders = new ArrayList<LoaderThread>();
-
-        loaders.add(new LoaderThread(srcDir + "person.jpg",
-                destDir + File.separator + "p%d.jpg", ScaleFactors.users));
-        loaders.add(new LoaderThread(srcDir + "person_thumb.jpg",
-                destDir + File.separator + "p%dt.jpg", ScaleFactors.users));
-        loaders.add(new LoaderThread(srcDir + "event.jpg",
-                destDir + File.separator + "e%d.jpg", ScaleFactors.events));
-        loaders.add(new LoaderThread(srcDir + "event_thumb.jpg",
-                destDir + File.separator + "e%dt.jpg", ScaleFactors.events));
-        loaders.add(new LoaderThread(srcDir + "event.pdf",
-                destDir + File.separator + "e%d.pdf", ScaleFactors.events));
 
-        for (LoaderThread loader : loaders) {
-            loader.join();
-        }
+        File f = new File(dest, "persons");
+        if (f.exists())
+            FileHelper.recursiveDelete(f);
+
+        f = new File(dest, "personThumbs");
+        if (f.exists())
+            FileHelper.recursiveDelete(f);
+
+        f = new File(dest, "events");
+        if (f.exists())
+            FileHelper.recursiveDelete(f);
+
+        f = new File(dest, "eventThumbs");
+        if (f.exists())
+            FileHelper.recursiveDelete(f);
+
+        f = new File(dest, "eventLits");
+        if (f.exists())
+            FileHelper.recursiveDelete(f);
+
+        threadPool = Executors.newFixedThreadPool(loaderThreads);
+
+        logger.info("Loading files to " + destDir);
+
+        load(srcDir + "person.jpg", destDir + File.separator +
+                "persons", "p%d.jpg", ScaleFactors.users);
+        load(srcDir + "person_thumb.jpg", destDir + File.separator +
+                "personThumbs", "p%dt.jpg", ScaleFactors.users);
+        load(srcDir + "event.jpg", destDir + File.separator +
+                "events", "e%d.jpg", ScaleFactors.events);
+        load(srcDir + "event_thumb.jpg", destDir + File.separator +
+                "eventThumbs", "e%dt.jpg", ScaleFactors.events);
+        load(srcDir + "event.pdf", destDir + File.separator +
+                "eventLits", "e%dl.pdf", ScaleFactors.events);
+
+        threadPool.shutdown();
+
+        boolean terminated = false;
+        while (!terminated)
+            try {
+                terminated = threadPool.awaitTermination(1, TimeUnit.HOURS);
+            } catch (InterruptedException e) {
+            }
 
-        for (LoaderThread loader : loaders) {
-            loader.close();
+        for (FileChannel channel : openFiles) {
+            channel.close();
         }
 
         System.exit(0);
-/*
-        FileChannel img = new FileInputStream(
-                                    srcDir + "person.jpg").getChannel();
-        FileChannel thumb = new FileInputStream(
-                                    srcDir + "person_thumb.jpg").getChannel();
-        long imgSize = img.size();
-        long thumbSize = thumb.size();
-
-        logger.info("Loading user images...");
-        for (int i = 1; i <= ScaleFactors.users; i++) {
-            logger.finer("Loading files for user " + i);
-            copyTo(img, imgSize, destDir + File.separator + "p" + i + ".jpg");
-            copyTo(thumb, thumbSize,
-                    destDir + File.separator + "p" + i + "t.jpg");
-        }
-
-        img.close();
-        thumb.close();
-
-        logger.info("Loading event images and files...");
-        img = new FileInputStream(srcDir + "event.jpg").getChannel();
-        thumb = new FileInputStream(srcDir + "event_thumb.jpg").getChannel();
-        FileChannel lit = new FileInputStream(
-                                    srcDir + "event.pdf").getChannel();
-
-        imgSize = img.size();
-        thumbSize = thumb.size();
-        long litSize = lit.size();
-
-        for (int i = 1; i <= ScaleFactors.events; i++) {
-            logger.finer("Loading files for event " + i);
-            copyTo(img, imgSize, destDir + File.separator + "e" + i + ".jpg");
-            copyTo(thumb, thumbSize,
-                    destDir + File.separator + "e" + i + "t.jpg");
-            copyTo(lit, litSize, destDir + File.separator + "e" + i + ".pdf");
-        }
-
-        img.close();
-        thumb.close();
-        lit.close();
-        System.exit(0);
-*/
     }
 
-    /*
-    private static void copyTo(FileChannel src, long size, String destFile)
-            throws IOException {
-        FileChannel dest = (new FileOutputStream(destFile)).getChannel();
-        src.transferTo(0, size, dest);
-        dest.close();
+    private static void load(String srcFile, String destDir,
+                             String destPattern, int count)
+            throws FileNotFoundException {
+        Formatter format = new Formatter();
+        FileChannel srcChannel = new FileInputStream(srcFile).getChannel();
+        openFiles.add(srcChannel);
+        for (int i = 1; i <= count; i++) {
+            int dir1 = i % 1000;
+            int dir2 = i / 1000000 % 1000;
+            String dirName =
+                    format.format(destDir + DIR_PATTERN, dir1, dir2).toString();
+            File dir = new File(dirName);
+            dir.mkdirs();
+            String dest = format.format(destPattern, i).toString();
+            if (i % REPORT_FREQUENCY == 0) {
+                logger.info("Copying to " + dest);
+            }
+            ((StringBuilder) format.out()).setLength(0);
+            threadPool.submit(loaderPool.getLoader(srcChannel, dest, i));
+        }
     }
-    */
 
-    static class LoaderThread extends Thread {
+
+    static class Loader implements Runnable {
 
         FileChannel src;
-        int count;
-        long size;
-        String pattern;
-        Formatter format;
-
-        public LoaderThread(String src, String destPattern, int count)
-                throws IOException {
-            this.src = new FileInputStream(src).getChannel();
-            size = this.src.size();
-            this.count = count;
-            this.pattern = destPattern;
-            format = new Formatter();
-            start();
-        }
+        String dest;
+        int fileNo;
 
         public void run() {
-            for (int i = 1; i <= count; i++) {
-                String dest = format.format(pattern, i).toString();
-                if (i % 1000 == 0) {
+            try {
+                FileChannel destChannel =
+                        new FileOutputStream(dest).getChannel();
+                src.transferTo(0, src.size(), destChannel);
+                destChannel.close();
+                if (fileNo % REPORT_FREQUENCY == 0)
                     logger.info("Copying to " + dest);
-                }
-                ((StringBuilder) format.out()).setLength(0);
-                try {
-                    FileChannel destChannel = new FileOutputStream(dest).
-                                                            getChannel();
-                    src.transferTo(0, size, destChannel);
-                    destChannel.close();
-                } catch (IOException e) {
-                    logger.log(Level.WARNING, "Error writing file " + dest, e);
+
+            } catch (IOException e) {
+                logger.log(Level.WARNING, "Error writing file " + dest, e);
+            } finally {
+                loaderPool.putLoader(this);
+            }
+        }
+    }
+
+    static class LoaderPool {
+
+        LinkedBlockingDeque<Loader> pool = new LinkedBlockingDeque<Loader>();
+        int count = 0;
+        int size;
+
+        public LoaderPool(int size) {
+            this.size = size;
+        }
+
+        public Loader getLoader(FileChannel src, String dest, int fileNo) {
+            Loader loader = pool.poll();
+            if (loader == null) {
+                if (count < size) {
+                    loader = new Loader();
+                    ++count;
+                } else {
+                    for (;;) {
+                    try {
+                        loader = pool.take();
+                        break;
+                    } catch (InterruptedException ex) {
+                        logger.log(Level.WARNING, "getLoader interrupted", ex);
+                    }
+                    }
                 }
             }
+            loader.src = src;
+            loader.dest = dest;
+            loader.fileNo = fileNo;
+            return loader;
         }
 
-        public void close() throws IOException {
-            src.close();
+        public void putLoader(Loader loader) {
+            for (;;) {
+                try {
+                    // User a LIFO model to keep the hot objects in cache.
+                    pool.putFirst(loader);
+                    break;
+                } catch (InterruptedException ex) {
+                    logger.log(Level.WARNING, "putLoader interrupted!", ex);
+                }
+            }
         }
     }
 }

Modified: incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Address.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Address.java?rev=831988&r1=831987&r2=831988&view=diff
==============================================================================
--- incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Address.java (original)
+++ incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Address.java Mon
Nov  2 18:08:41 2009
@@ -108,7 +108,7 @@
         ThreadConnection c = ThreadConnection.getInstance();
         try {
             //update id
-             logger.info("Updating Address ID");
+             logger.fine("Updating Address ID");
               
              c.prepareStatement("INSERT INTO ID_GEN " +
                     "(GEN_KEY, GEN_VALUE) " +
@@ -121,7 +121,7 @@
                     "where GEN_KEY='ADDRESS_ID'");
              c.executeUpdate();
              */ 
-             logger.info("After updating Address ID");
+             logger.fine("After updating Address ID");
         } catch (SQLException e) {
             logger.log(Level.SEVERE, e.getMessage(), e);
             //LoadController.increaseErrorCount();

Modified: incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Friends.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Friends.java?rev=831988&r1=831987&r2=831988&view=diff
==============================================================================
--- incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Friends.java (original)
+++ incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Friends.java Mon
Nov  2 18:08:41 2009
@@ -37,27 +37,27 @@
     // We use on average of 15 friends. Random 2..28 Friends.
 
     private static final String STATEMENT = "insert into PERSON_PERSON " +
-            "(Person_username, friends_username) values (?, ?)";
+            "(Person_userid, friends_userid) values (?, ?)";
 
     static Logger logger = Logger.getLogger(Friends.class.getName());
 
-    //int id;
-    String userName;
-    String[] friends;
-
+    int id;
+    //String userName;
+    //String[] friends;
+    LinkedHashSet<Integer> friendSet;
     public String getClearStatement() {
         return "truncate table PERSON_PERSON";
     }
 
     public void prepare() {
-        int id = getSequence();
+        id = getSequence();
         ++id;
         ThreadResource tr = ThreadResource.getInstance();
         Random r = tr.getRandom();
-        userName = UserName.getUserName(id);
+        //userName = UserName.getUserName(id);
         int count = r.random(2, 28);
+	    friendSet = new LinkedHashSet<Integer>(count);
 
-        LinkedHashSet<Integer> friendSet = new LinkedHashSet<Integer>(count);
         for (int i = 0; i < count; i++) {
             int friendId;
             do { // Prevent friend to be the same user.
@@ -65,19 +65,22 @@
             } while (friendId == id || !friendSet.add(friendId));
         }
 
-        friends = new String[friendSet.size()];
-        int idx = 0;
-        for (int friendId : friendSet)
-            friends[idx++] = UserName.getUserName(friendId);
+        //friends = new String[friendSet.size()];
+        //int idx = 0;
+        //for (int friendId : friendSet)
+         //   friends[idx++] = UserName.getUserName(friendId);
     }
 
     public void load() {
         ThreadConnection c = ThreadConnection.getInstance();
         try {
-            for (String friend : friends) {
+        //    for (String friend : friends) {
+			  for (Integer friend : friendSet) {
                 PreparedStatement s = c.prepareStatement(STATEMENT);
-                s.setString(1, userName);
-                s.setString(2, friend);
+                //s.setString(1, userName);
+                //s.setString(2, friend);
+				s.setInt(1, id);
+				s.setInt(2, friend);
                 c.addBatch();
             }
         } catch (SQLException e) {

Modified: incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/LoadController.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/LoadController.java?rev=831988&r1=831987&r2=831988&view=diff
==============================================================================
--- incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/LoadController.java
(original)
+++ incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/LoadController.java
Mon Nov  2 18:08:41 2009
@@ -72,6 +72,7 @@
         // This is to ensure all load tasks are done before this one starts.
         postLoad(Tag.class);
         postLoad(Address.class);
+        postLoad(Person.class);
         postLoad(SocialEvent.class);
         postLoad(Comments.class);
         postLoad(Invitation.class);

Modified: incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Person.java
URL: http://svn.apache.org/viewvc/incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Person.java?rev=831988&r1=831987&r2=831988&view=diff
==============================================================================
--- incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Person.java (original)
+++ incubator/olio/workload/java/trunk/src/org/apache/olio/workload/loader/Person.java Mon
Nov  2 18:08:41 2009
@@ -83,4 +83,25 @@
             logger.log(Level.SEVERE, e.getMessage(), e);
         }
     }
+     /**
+     * Update ID after all the data is loaded.
+     * So we update the ID_GEN table at postload and add 1 to count.
+     */
+    public void postLoad() {
+        ThreadConnection c = ThreadConnection.getInstance();
+        try {
+            //update id
+             logger.fine("Updating Person ID");
+              
+             c.prepareStatement("INSERT INTO ID_GEN " +
+                    "(GEN_KEY, GEN_VALUE) " +
+                    "VALUES ('PERSON_ID', "+ ScaleFactors.users+1  + ")");
+             c.executeUpdate();
+        } catch (SQLException e) {
+            logger.log(Level.SEVERE, e.getMessage(), e);
+            //LoadController.increaseErrorCount();
+        }
+
+
+    }
 }



Mime
View raw message