camel-users mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Rafal Janik <rafal.ja...@softwaremind.pl>
Subject Large files processing
Date Mon, 20 Jul 2009 11:44:55 GMT
Hi - i have problem with big files processing in camel.

I've got osgi bundle in FuseESB (servicemix) which is using the camel .
In camel-context.xml file there  is :

    <route>
            <from 
uri="file://inbox/upload-file-simple?consumer.delay=2000&amp;consumer.initialDelay=2000&amp;noop=true"

/>
            <to uri="..." />           
            <choice>
                <when>
                    
<groovy>Integer.parseInt(request.headers['file-size']) > 20000</groovy>
                    <to uri="A..." />
                </when>
                <otherwise>
                    <to uri="B" />
                </otherwise>
            </choice>
        </route>



The result depends on the delay.  If  the delay is to short  only a part 
of the file is sent to "next uri".

To check it i wrote a simple application:

package test.camel;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

import org.apache.camel.CamelContext;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.impl.DefaultCamelContext;

public class CamelTest {

    private final static String TEST_FILE_NAME = "test.dat";
    private final static String TEST_FILE_DIR  = "src/main/resources";
    private final static String INBOX_DIR      = "inbox";
    private final static String OUTBOX_DIR     = "outbox";

    /**
     * @param args
     * @throws Exception
     * @throws Exception
     */
    public static void main(String[] args) {

        System.out.println("Camel store file test start");
        CamelContext context;
        try {
            context = getCamelContext();
            context.start();
            copyFileToInbox();
        } catch (FileNotFoundException e) {
            System.out.println("Cannot fonde test file " + TEST_FILE_NAME);
        } catch (Exception e) {
            System.out.println("contextStart Error:");
            e.printStackTrace();
        }
        boolean work = true;
        while (work) {
        }
    }

    public static CamelContext getCamelContext() {
        System.out.println("Start congire Camel... ");
        CamelContext context = new DefaultCamelContext();
        try {
            context.addRoutes(new RouteBuilder() {
                public void configure() {
                    from("file://" + INBOX_DIR + 
"?noop=true").to("file://" + OUTBOX_DIR);
                    from("file://" + OUTBOX_DIR + 
"?noop=true").process(new Processor() {

                        public void process(Exchange e) {
                            System.out.println("Received in outbox: " + 
e.getIn());
                        }
                    });
                }
            });
        } catch (Exception e) {
            System.out.println("getCamelContext Error:");
            e.printStackTrace();

        }
        return context;
    }

    public static void copyFileToInbox() throws FileNotFoundException {

        File file = new File(TEST_FILE_DIR, TEST_FILE_NAME);
        System.out.println("File  " + file.getName() + " exists: " + 
file.exists() + " (" + file.length()
                           + "b)");

        File f2 = new File(INBOX_DIR, TEST_FILE_NAME);
        InputStream in = new FileInputStream(file);
        OutputStream out = new FileOutputStream(f2);

        byte[] buf = new byte[1024];
        int len;
        try {
            while ((len = in.read(buf)) > 0) {
                out.write(buf, 0, len);
            }
        } catch (IOException e) {
            System.out.println("copyFileToInbox Error:");
            e.printStackTrace();
        } finally {
            try {
                in.close();
                out.close();
            } catch (IOException e) {
                System.out.println("Error while closing streams");
            }
        }
    }
}


And the resul was similiar - while uploading large files (about 
100-300MB) in outbox folder there was only a part of orginal file,
so i tried to set  "noop=true" and the result was surprising - in outbox 
folder i got the much more bigger file than was sent to inbox.

What options should be set to process the whole file?
What i do wrong?

Best Regards,
rafal



Mime
View raw message