From olio-commits-return-101-apmail-incubator-olio-commits-archive=incubator.apache.org@incubator.apache.org Wed Mar 11 21:45:02 2009 Return-Path: Delivered-To: apmail-incubator-olio-commits-archive@minotaur.apache.org Received: (qmail 12880 invoked from network); 11 Mar 2009 21:45:02 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 11 Mar 2009 21:45:02 -0000 Received: (qmail 46337 invoked by uid 500); 11 Mar 2009 21:45:02 -0000 Delivered-To: apmail-incubator-olio-commits-archive@incubator.apache.org Received: (qmail 46323 invoked by uid 500); 11 Mar 2009 21:45:01 -0000 Mailing-List: contact olio-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: olio-dev@incubator.apache.org Delivered-To: mailing list olio-commits@incubator.apache.org Received: (qmail 46312 invoked by uid 99); 11 Mar 2009 21:45:01 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 11 Mar 2009 14:45:01 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 11 Mar 2009 21:44:50 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 2AE192388965; Wed, 11 Mar 2009 21:44:29 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r752637 - /incubator/olio/workload/rails/trunk/src/org/apache/olio/workload/fsloader/FileLoader.java Date: Wed, 11 Mar 2009 21:44:29 -0000 To: olio-commits@incubator.apache.org From: shanti@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20090311214429.2AE192388965@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: shanti Date: Wed Mar 11 21:44:28 2009 New Revision: 752637 URL: http://svn.apache.org/viewvc?rev=752637&view=rev Log: Fixes for OLIO-66, OLIO-68, OLIO-60 OLIO-60 still needs further fixes to the dbloader. All fixes were done by simply copying over the php file loader code. Modified: incubator/olio/workload/rails/trunk/src/org/apache/olio/workload/fsloader/FileLoader.java Modified: incubator/olio/workload/rails/trunk/src/org/apache/olio/workload/fsloader/FileLoader.java URL: http://svn.apache.org/viewvc/incubator/olio/workload/rails/trunk/src/org/apache/olio/workload/fsloader/FileLoader.java?rev=752637&r1=752636&r2=752637&view=diff ============================================================================== --- incubator/olio/workload/rails/trunk/src/org/apache/olio/workload/fsloader/FileLoader.java (original) +++ incubator/olio/workload/rails/trunk/src/org/apache/olio/workload/fsloader/FileLoader.java Wed Mar 11 21:44:28 2009 @@ -16,23 +16,79 @@ * limitations under the License. * */ + package org.apache.olio.workload.fsloader; import org.apache.olio.workload.util.ScaleFactors; -import java.nio.channels.FileChannel; -import java.io.FileInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.channels.FileChannel; +import java.util.Formatter; +import java.util.ArrayList; +import java.util.logging.Level; +import java.util.logging.Logger; public class FileLoader { + private static Logger logger = Logger.getLogger(FileLoader.class.getName()); + public static void main(String[] args) throws Exception { String srcDir = args[0]; - ScaleFactors.setActiveUsers(Integer.parseInt(args[1])); + String destDir = args[1]; + ScaleFactors.setActiveUsers(Integer.parseInt(args[2])); srcDir += File.separator; + + // Clear the dest dir + File dest = new File(destDir); + if (!dest.isDirectory()) { + logger.severe(destDir + " Not a directory!"); + System.exit(1); + } + + logger.info("Deleting files in " + destDir); + File[] list = dest.listFiles(); + for (File f : list) { + String name = f.getName(); + boolean delete = false; + if (name.endsWith(".jpg")) + delete = true; + else if (name.endsWith(".JPG")) + delete = true; + else if (name.endsWith(".pdf")) + delete = true; + else if (name.endsWith(".PDF")) + delete = true; + if (delete && !f.delete()) + logger.warning("Error deleting file " + f.getName()); + } + + ArrayList loaders = new ArrayList(); + + loaders.add(new LoaderThread(srcDir + "person.jpg", + destDir + File.separator + "p%d.jpg", ScaleFactors.users)); + loaders.add(new LoaderThread(srcDir + "person_thumb.jpg", + destDir + File.separator + "p%dt.jpg", ScaleFactors.users)); + loaders.add(new LoaderThread(srcDir + "event.jpg", + destDir + File.separator + "e%d.jpg", ScaleFactors.events)); + loaders.add(new LoaderThread(srcDir + "event_thumb.jpg", + destDir + File.separator + "e%dt.jpg", ScaleFactors.events)); + loaders.add(new LoaderThread(srcDir + "event.pdf", + destDir + File.separator + "e%d.pdf", ScaleFactors.events)); + + for (LoaderThread loader : loaders) { + loader.join(); + } + + for (LoaderThread loader : loaders) { + loader.close(); + } + + System.exit(0); +/* FileChannel img = new FileInputStream( srcDir + "person.jpg").getChannel(); FileChannel thumb = new FileInputStream( @@ -40,16 +96,18 @@ long imgSize = img.size(); long thumbSize = thumb.size(); + logger.info("Loading user images..."); for (int i = 1; i <= ScaleFactors.users; i++) { - System.out.println("Loading files for user " + i); - copyTo(img, imgSize, "p" + i + ".jpg"); - copyTo(thumb, thumbSize, "p" + i + "_thumb.jpg"); + logger.finer("Loading files for user " + i); + copyTo(img, imgSize, destDir + File.separator + "p" + i + ".jpg"); + copyTo(thumb, thumbSize, + destDir + File.separator + "p" + i + "t.jpg"); } img.close(); thumb.close(); - + logger.info("Loading event images and files..."); img = new FileInputStream(srcDir + "event.jpg").getChannel(); thumb = new FileInputStream(srcDir + "event_thumb.jpg").getChannel(); FileChannel lit = new FileInputStream( @@ -60,22 +118,67 @@ long litSize = lit.size(); for (int i = 1; i <= ScaleFactors.events; i++) { - System.out.println("Loading files for event " + i); - copyTo(img, imgSize, "e" + i + ".jpg"); - copyTo(thumb, thumbSize, "e" + i + "_thumb.jpg"); - copyTo(lit, litSize, "e" + i + ".pdf"); + logger.finer("Loading files for event " + i); + copyTo(img, imgSize, destDir + File.separator + "e" + i + ".jpg"); + copyTo(thumb, thumbSize, + destDir + File.separator + "e" + i + "t.jpg"); + copyTo(lit, litSize, destDir + File.separator + "e" + i + ".pdf"); } img.close(); thumb.close(); lit.close(); System.exit(0); +*/ } + /* private static void copyTo(FileChannel src, long size, String destFile) throws IOException { FileChannel dest = (new FileOutputStream(destFile)).getChannel(); src.transferTo(0, size, dest); dest.close(); } + */ + + static class LoaderThread extends Thread { + + FileChannel src; + int count; + long size; + String pattern; + Formatter format; + + public LoaderThread(String src, String destPattern, int count) + throws IOException { + this.src = new FileInputStream(src).getChannel(); + size = this.src.size(); + this.count = count; + this.pattern = destPattern; + format = new Formatter(); + start(); + } + + public void run() { + for (int i = 1; i <= count; i++) { + String dest = format.format(pattern, i).toString(); + if (i % 1000 == 0) { + logger.info("Copying to " + dest); + } + ((StringBuilder) format.out()).setLength(0); + try { + FileChannel destChannel = new FileOutputStream(dest). + getChannel(); + src.transferTo(0, size, destChannel); + destChannel.close(); + } catch (IOException e) { + logger.log(Level.WARNING, "Error writing file " + dest, e); + } + } + } + + public void close() throws IOException { + src.close(); + } + } }