jackrabbit-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Stefan Guggisberg <stefan.guggisb...@gmail.com>
Subject Re: Performance question
Date Tue, 12 Jul 2005 16:36:54 GMT
hi jeff

On 7/12/05, Jeff Foster <jeff_c_foster@yahoo.co.uk> wrote:
> Hi
> 
> I have been giving JackRabbit a go and was interested in the performance.
> I have seen some claims about performance (1 million nodes per hour) but have not been
able to get anything like this.
> 
> So I have written a little util to upload files and folders and then loaded up about
4000 file and folders out of a java project (mostly java files).
> 
> So on my somewhat out of date laptop (Dell Lattitude C840 2G 1800 Mhz Windows XP)
> I get the following
> 
> With the file system:
> 1  785 s
> 2 968 s
> 
> Again:
> 1 899 s
> 2 1075 s
> 
> And with all those files ... and the time to delete them I gave up..
> 
> CQFS
> 1 295.4 s
> 2 378.6 s
> 3 409.9 s
> 4 453.4 s
> 5 544.0 s
> 6 10637 s
> 7 1542.2 s
> 
> I accept the windows file system version is limited and my laptop is limited (especially
waiting for all those files to be deleted!)
> 
> But CQFS seems to only give me 13 files/folders per second at best and this performance
seems to tail off quite quickly. Am I doing something silly in the configuration? How many
nodes would be created per file and per folder?

a folder translates to 1 node and 2 properties. a file translates to 2
nodes and 8 properties.
the poor performance you're experiencing is caused by the windows
filesystem i am afraid
(even when using CQFS). the contents of every file is represented as a
BINARY property which in turn is internally backed by a file in the
local filesystem. if you run your test on a linux box you'll be seeing
a lot better results with both local filesystem and CQFS.
on the other hand, if you run a test that is generating about the same
amount of items but doesn't use BINARY properties you'll see a
dramatic difference.

to improve the performance you could either
- minimze/avoid use of BINARY properties
- use an OS with a decent filesystem
- write a custom persistence manager that 'inlines' BINARY properties
  (e.g. using a size threshold) rather than storing them in the local filesystem

cheers
stefan

> 
> It also took me a while to realise I needed to be in the correct place to download the
dependancies via mavern.
> 
> Cheers
> 
> Jeff
> 
> 
> 
> 
> 
> 
> 
> 
> 
> 
> 
> import java.io.BufferedInputStream;
> import java.io.File;
> import java.io.FileFilter;
> import java.io.FileInputStream;
> import java.io.FileNotFoundException;
> import java.io.Serializable;
> import java.util.Calendar;
> import java.util.Date;
> import java.util.GregorianCalendar;
> import java.util.HashMap;
> import java.util.Hashtable;
> import java.util.Map;
> import javax.jcr.*;
> import javax.jcr.lock.LockException;
> import javax.jcr.nodetype.ConstraintViolationException;
> import javax.jcr.nodetype.NoSuchNodeTypeException;
> import javax.jcr.version.VersionException;
> import javax.naming.Context;
> import javax.naming.InitialContext;
> import org.apache.jackrabbit.core.SearchManager;
> import org.apache.jackrabbit.core.jndi.RegistryHelper;
> import org.apache.jackrabbit.value.DateValue;
> import org.apache.jackrabbit.value.StringValue;
> public class JackRabbitFileLoader
> {
>     public JackRabbitFileLoader()
>     {
>         super();
>     }
> 
>     private Node create(Node container, File file, FileFilter filter, boolean recurse)
 throws Exception
>     {
>         if (file.isDirectory())
>         {
>             Node directoryNodeRef = createDirectory(container, file);
> 
>             if(recurse)
>             {
>                 File[] files = ((filter == null) ? file.listFiles() : file.listFiles(filter));
>                 for(int i = 0; i < files.length; i++)
>                 {
>                     create(directoryNodeRef, files[i], filter, recurse);
>                 }
>             }
> 
>             return directoryNodeRef;
>         }
>         else
>         {
>             return createFile(container, file);
>         }
>     }
>     private Node createFile(Node parentNode, File file) throws Exception
>     {
>         Calendar cal = new GregorianCalendar();
>         Node nodeRef = parentNode.addNode(file.getName(), "nt:file");
>         // create properties for content type
> 
>         Node resource = nodeRef.addNode("jcr:content", "nt:resource");
> 
>         resource.setProperty("jcr:encoding", "UTF-8");
>         resource.setProperty("jcr:mimeType", "text/plain");
>         resource.setProperty("jcr:lastModified", new DateValue(cal));
>         // apply the titled aspect - title and description
> 
>         resource.setProperty("jcr:data", new BufferedInputStream(new FileInputStream(file)));
>         return nodeRef;
>     }
>     private Node createDirectory(Node parentNode, File file)  throws Exception
>     {
>         Node nodeRef = parentNode.addNode(file.getName(), "nt:folder");
> 
>         return nodeRef;
>     }
>     public static void main(String[] args) throws Exception
>     {
>         try
>         {
> 
>             JackRabbitFileLoader fileLoader = new JackRabbitFileLoader();
> 
>             //String configFile = "repotest/repository-file.xml";
>             String configFile = "repotest/repository.xml";
> 
>             String repHomeDir = "repotest";
>             Hashtable env = new Hashtable();
>             env.put(Context.INITIAL_CONTEXT_FACTORY, "org.apache.jackrabbit.core.jndi.provider.DummyInitialContextFactory");
>             env.put(Context.PROVIDER_URL, "localhost");
>             InitialContext ctx = new InitialContext(env);
>             RegistryHelper.registerRepository(ctx, "repo", configFile, repHomeDir, true);
>             Repository r = (Repository) ctx.lookup("repo");
>             Session session = r.login(new SimpleCredentials("userid", "".toCharArray()),
null);
>             Node rn = session.getRootNode();
> 
> //            for(NodeIterator it = rn.getNodes(); it.hasNext(); /**/)
> //            {
> //                Node node = it.nextNode();
> //                deleteNode(node);
> //            }
> 
>             System.out.println(rn.getNodes().getSize());
> 
>             long start = System.nanoTime();
>             fileLoader.create(rn, new File(args[0]), null, true);
>             session.save();
>             long end = System.nanoTime();
>             System.out.println("Done in " +((end-start)/1000000.0f));
> 
>             System.out.println(rn.getNodes().getSize());
> 
>             //printNodes(rn, 0);
>         }
>         catch (Exception e)
>         {
>             e.printStackTrace();
>         }
>     }
> 
> 
>     private static void deleteNode(Node container) throws Exception
>     {
>         if(container.getDefinition().isProtected())
>         {
>             return;
>         }
>         for(NodeIterator it = container.getNodes(); it.hasNext(); /**/)
>         {
>             Node node = it.nextNode();
>             deleteNode(node);
>         }
>         container.remove();
>     }
> 
>     private static void printNodes(Node container, int indent) throws Exception
>     {
>         System.out.println(container.getPath());
>         for(NodeIterator it = container.getNodes(); it.hasNext(); /**/)
>         {
>             Node node = it.nextNode();
>             printNodes(node, indent+2);
>         }
>     }
> }
> 
> 
> 
> 
> 
> 
> 
> 
> <?xml version="1.0" encoding="ISO-8859-1"?>
> 
> <Repository>
> 
> <FileSystem class="org.apache.jackrabbit.core.fs.local.LocalFileSystem">
> 
> <param name="path" value="w:/cqfs/repository"/>
> 
> </FileSystem>
> 
> <Security appName="Jackrabbit">
> 
> <AccessManager class="org.apache.jackrabbit.core.security.SimpleAccessManager">
> 
> <!-- <param name="config" value="w:/cqfs/access.xml"/> -->
> 
> </AccessManager>
> 
> </Security>
> 
> <!--
> 
> location of workspaces root directory and name of default workspace
> 
> -->
> 
> <Workspaces rootPath="w:/cqfs/workspaces" defaultWorkspace="default"/>
> 
> <Workspace name="${wsp.name}">
> 
> <FileSystem class="com.day.jackrabbit.fs.cq.CQFileSystem">
> 
> <param name="path" value="${wsp.home}/wspStore.dat"/>
> 
> <param name="autoRepair" value="false"/>
> 
> <param name="blockSize" value="128"/>
> 
> <param name="autoSync" value="false"/>
> 
> </FileSystem>
> 
> <!--
> 
> <FileSystem class="org.apache.jackrabbit.core.fs.local.LocalFileSystem">
> 
> <param name="path" value="${wsp.home}"/>
> 
> </FileSystem>
> 
> -->
> 
> <!--
> 
> persistence manager of the workspace:
> 
> class: FQN of class implementing the PersistenceManager interface
> 
> -->
> 
> <!--
> 
> <PersistenceManager class="org.apache.jackrabbit.core.state.xml.XMLPersistenceManager"/>
> 
> -->
> 
> <!--
> 
> <PersistenceManager class="org.apache.jackrabbit.core.state.mem.InMemPersistenceManager">
> 
> <param name="initialCapacity" value="100000"/>
> 
> <param name="loadFactor" value="0.3"/>
> 
> <param name="persistent" value="true"/>
> 
> </PersistenceManager>
> 
> -->
> 
> <PersistenceManager class="org.apache.jackrabbit.core.state.obj.ObjectPersistenceManager"/>
> 
> <!--
> 
> Search index and the file system it uses.
> 
> -->
> 
> <SearchIndex class="org.apache.jackrabbit.core.query.lucene.SearchIndex">
> 
> <param name="useCompoundFile" value="true"/>
> 
> <param name="minMergeDocs" value="1000"/>
> 
> <param name="maxMergeDocs" value="10000"/>
> 
> <param name="mergeFactor" value="10"/>
> 
> <FileSystem class="org.apache.jackrabbit.core.fs.local.LocalFileSystem">
> 
> <param name="path" value="${wsp.home}/index"/>
> 
> </FileSystem>
> 
> </SearchIndex>
> 
> </Workspace>
> 
> <!--
> 
> Configures the versioning
> 
> -->
> 
> <Versioning rootPath="w:/cqfs/version">
> 
> <!--
> 
> Configures the filesystem to use for versioning for the respective
> 
> persistence manager
> 
> -->
> 
> <FileSystem class="com.day.jackrabbit.fs.cq.CQFileSystem">
> 
> <param name="path" value="w:/cqfs/version/version.dat"/>
> 
> <param name="autoRepair" value="false"/>
> 
> <param name="blockSize" value="128"/>
> 
> <param name="autoSync" value="false"/>
> 
> </FileSystem>
> 
> 
> 
> <!--
> 
> <FileSystem class="org.apache.jackrabbit.core.fs.local.LocalFileSystem">
> 
> <param name="path" value="w:/cqfs/version"/>
> 
> </FileSystem>
> 
> -->
> 
> 
> 
> <!--
> 
> Configures the perisistence manager to be used for persisting version state.
> 
> Please note that the current versioning implementation is based on
> 
> a 'normal' persistence manager, but this could change in future
> 
> implementations.
> 
> -->
> 
> <PersistenceManager class="org.apache.jackrabbit.core.state.obj.ObjectPersistenceManager"/>
> 
> </Versioning>
> 
> </Repository>
> 
> 
> 
> 
> 
> ---------------------------------
> Yahoo! Messenger NEW - crystal clear PC to PCcalling worldwide with voicemail
>

Mime
View raw message