chemistry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From f...@apache.org
Subject svn commit: r1204024 - in /chemistry/opencmis/trunk: chemistry-opencmis-server/chemistry-opencmis-server-bindings/src/main/java/org/apache/chemistry/opencmis/server/impl/atompub/ chemistry-opencmis-test/chemistry-opencmis-test-fit/src/test/java/org/apa...
Date Sat, 19 Nov 2011 17:25:50 GMT
Author: fmui
Date: Sat Nov 19 17:25:49 2011
New Revision: 1204024

URL: http://svn.apache.org/viewvc?rev=1204024&view=rev
Log:
CMIS-472: big content is now buffered in a temporary file (similar to the Web Services binding
implementation)

Modified:
    chemistry/opencmis/trunk/chemistry-opencmis-server/chemistry-opencmis-server-bindings/src/main/java/org/apache/chemistry/opencmis/server/impl/atompub/AtomEntryParser.java
    chemistry/opencmis/trunk/chemistry-opencmis-test/chemistry-opencmis-test-fit/src/test/java/org/apache/chemistry/opencmis/fit/runtime/AbstractWriteObjectIT.java

Modified: chemistry/opencmis/trunk/chemistry-opencmis-server/chemistry-opencmis-server-bindings/src/main/java/org/apache/chemistry/opencmis/server/impl/atompub/AtomEntryParser.java
URL: http://svn.apache.org/viewvc/chemistry/opencmis/trunk/chemistry-opencmis-server/chemistry-opencmis-server-bindings/src/main/java/org/apache/chemistry/opencmis/server/impl/atompub/AtomEntryParser.java?rev=1204024&r1=1204023&r2=1204024&view=diff
==============================================================================
--- chemistry/opencmis/trunk/chemistry-opencmis-server/chemistry-opencmis-server-bindings/src/main/java/org/apache/chemistry/opencmis/server/impl/atompub/AtomEntryParser.java
(original)
+++ chemistry/opencmis/trunk/chemistry-opencmis-server/chemistry-opencmis-server-bindings/src/main/java/org/apache/chemistry/opencmis/server/impl/atompub/AtomEntryParser.java
Sat Nov 19 17:25:49 2011
@@ -18,8 +18,14 @@
  */
 package org.apache.chemistry.opencmis.server.impl.atompub;
 
+import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -292,9 +298,9 @@ public class AtomEntryParser {
         } else if (type.startsWith("text/")) {
             bytes = readText(parser).getBytes("UTF-8");
         } else {
-            LightByteArrayOutputStream lbs = readBase64(parser);
-            atomContentStream.setStream(lbs.getInputStream());
-            atomContentStream.setLength(BigInteger.valueOf(lbs.getSize()));
+            ThresholdOutputStream ths = readBase64(parser);
+            atomContentStream.setStream(ths.getInputStream());
+            atomContentStream.setLength(BigInteger.valueOf(ths.getSize()));
         }
 
         if (bytes != null) {
@@ -321,9 +327,9 @@ public class AtomEntryParser {
                     if (TAG_MEDIATYPE.equals(name.getLocalPart())) {
                         cmisContentStream.setMimeType(readText(parser));
                     } else if (TAG_BASE64.equals(name.getLocalPart())) {
-                        LightByteArrayOutputStream lbs = readBase64(parser);
-                        cmisContentStream.setStream(lbs.getInputStream());
-                        cmisContentStream.setLength(BigInteger.valueOf(lbs.getSize()));
+                        ThresholdOutputStream ths = readBase64(parser);
+                        cmisContentStream.setStream(ths.getInputStream());
+                        cmisContentStream.setLength(BigInteger.valueOf(ths.getSize()));
                     } else {
                         skip(parser);
                     }
@@ -376,8 +382,8 @@ public class AtomEntryParser {
     /**
      * Parses a tag that contains base64 encoded content.
      */
-    private static LightByteArrayOutputStream readBase64(XMLStreamReader parser) throws Exception
{
-        LightByteArrayOutputStream bufferStream = new LightByteArrayOutputStream();
+    private static ThresholdOutputStream readBase64(XMLStreamReader parser) throws Exception
{
+        ThresholdOutputStream bufferStream = new ThresholdOutputStream();
         Base64.OutputStream b64stream = new Base64.OutputStream(bufferStream, Base64.DECODE);
 
         next(parser);
@@ -400,6 +406,8 @@ public class AtomEntryParser {
             }
         }
 
+        b64stream.close();
+
         next(parser);
 
         return bufferStream;
@@ -564,114 +572,228 @@ public class AtomEntryParser {
         return false;
     }
 
-    private static class LightByteArrayOutputStream extends OutputStream {
+    private static class ThresholdOutputStream extends OutputStream {
+
         private static final int MAX_GROW = 10 * 1024 * 1024;
+        // TODO: make threshold configurable
+        private static final int THRESHOLD = 4 * 1024 * 1024;
 
         private byte[] buf = null;
-        private int size = 0;
+        private int bufSize = 0;
+        private long size;
+        private File tmpFile;
+        private OutputStream tmpStream;
 
-        public LightByteArrayOutputStream() {
+        public ThresholdOutputStream() {
             this(64 * 1024);
         }
 
-        public LightByteArrayOutputStream(int initSize) {
+        public ThresholdOutputStream(int initSize) {
             if (initSize < 0) {
                 throw new IllegalArgumentException("Negative initial size: " + initSize);
             }
             buf = new byte[initSize];
         }
 
-        private void expand(int i) {
-            if (size + i <= buf.length) {
+        private void expand(int nextBufferSize) throws IOException {
+            if (bufSize + nextBufferSize <= buf.length) {
                 return;
             }
 
-            int newSize = ((size + i) * 2 < MAX_GROW ? (size + i) * 2 : buf.length + i
+ MAX_GROW);
+            if (bufSize + nextBufferSize > THRESHOLD) {
+                if (tmpStream == null) {
+                    tmpFile = File.createTempFile("opencmis", null);
+                    tmpStream = new FileOutputStream(tmpFile);
+                }
+                tmpStream.write(buf, 0, bufSize);
+
+                if (buf.length != THRESHOLD) {
+                    buf = new byte[THRESHOLD];
+                }
+                bufSize = 0;
+
+                return;
+            }
+
+            int newSize = ((bufSize + nextBufferSize) * 2 < MAX_GROW ? (bufSize + nextBufferSize)
* 2 : buf.length
+                    + nextBufferSize + MAX_GROW);
             byte[] newbuf = new byte[newSize];
-            System.arraycopy(buf, 0, newbuf, 0, size);
+            System.arraycopy(buf, 0, newbuf, 0, bufSize);
             buf = newbuf;
         }
 
-        public int getSize() {
+        public long getSize() {
             return size;
         }
 
         @Override
-        public void write(byte[] buffer) {
+        public void write(byte[] buffer) throws IOException {
             write(buffer, 0, buffer.length);
         }
 
         @Override
-        public synchronized void write(byte[] buffer, int offset, int len) {
+        public void write(byte[] buffer, int offset, int len) throws IOException {
             if (len == 0) {
                 return;
             }
 
             expand(len);
-            System.arraycopy(buffer, offset, buf, size, len);
+            System.arraycopy(buffer, offset, buf, bufSize, len);
+            bufSize += len;
             size += len;
         }
 
         @Override
-        public void write(int oneByte) {
-            if (size == buf.length) {
+        public void write(int oneByte) throws IOException {
+            if (bufSize == buf.length) {
                 expand(1);
             }
 
-            buf[size++] = (byte) oneByte;
+            buf[bufSize++] = (byte) oneByte;
+            size++;
+        }
+
+        @Override
+        public void flush() throws IOException {
+            if (tmpStream != null) {
+                if (bufSize > 0) {
+                    tmpStream.write(buf, 0, bufSize);
+                    bufSize = 0;
+                }
+                tmpStream.flush();
+            }
+        }
+
+        @Override
+        public void close() throws IOException {
+            flush();
+
+            if (tmpStream != null) {
+                tmpStream.close();
+            }
         }
 
-        public InputStream getInputStream() {
-            return new InputStream() {
+        public InputStream getInputStream() throws Exception {
+            if (tmpStream != null) {
+                close();
+                buf = null;
+
+                return new InternalTempFileInputStream();
+            } else {
+                return new InternalBufferInputStream();
+            }
+        }
+
+        private class InternalBufferInputStream extends InputStream {
+
+            private int pos = 0;
+
+            @Override
+            public boolean markSupported() {
+                return false;
+            }
+
+            @Override
+            public int available() {
+                return bufSize - pos;
+            }
+
+            @Override
+            public int read() {
+                return (pos < bufSize) && (buf != null) ? (buf[pos++] & 0xff)
: -1;
+            }
 
-                private int pos = 0;
+            @Override
+            public int read(byte[] b) throws IOException {
+                return read(b, 0, b.length);
+            }
 
-                @Override
-                public int available() {
-                    return size - pos;
+            @Override
+            public int read(byte[] b, int off, int len) {
+                if ((pos >= bufSize) || (buf == null)) {
+                    return -1;
                 }
 
-                @Override
-                public int read() {
-                    return (pos < size) && (buf != null) ? (buf[pos++] & 0xff)
: -1;
+                if ((pos + len) > bufSize) {
+                    len = (bufSize - pos);
                 }
 
-                @Override
-                public int read(byte[] b, int off, int len) {
-                    if ((pos >= size) || (buf == null)) {
-                        return -1;
-                    }
+                System.arraycopy(buf, pos, b, off, len);
+                pos += len;
 
-                    if ((pos + len) > size) {
-                        len = (size - pos);
-                    }
+                return len;
+            }
 
-                    System.arraycopy(buf, pos, b, off, len);
-                    pos += len;
+            @Override
+            public long skip(long n) {
+                if ((pos + n) > bufSize) {
+                    n = bufSize - pos;
+                }
 
-                    return len;
+                if (n < 0) {
+                    return 0;
                 }
 
-                @Override
-                public long skip(long n) {
-                    if ((pos + n) > size) {
-                        n = size - pos;
-                    }
+                pos += n;
 
-                    if (n < 0) {
-                        return 0;
-                    }
+                return n;
+            }
+
+            @Override
+            public void close() throws IOException {
+                buf = null;
+            }
+        }
+
+        private class InternalTempFileInputStream extends FilterInputStream {
+
+            private boolean isDeleted = false;
 
-                    pos += n;
+            public InternalTempFileInputStream() throws FileNotFoundException {
+                super(new BufferedInputStream(new FileInputStream(tmpFile), THRESHOLD));
+            }
+
+            @Override
+            public boolean markSupported() {
+                return false;
+            }
 
-                    return n;
+            @Override
+            public int read() throws IOException {
+                int b = super.read();
+
+                if (b == -1 && !isDeleted) {
+                    super.close();
+                    isDeleted = tmpFile.delete();
                 }
 
-                @Override
-                public void close() throws IOException {
-                    buf = null;
+                return b;
+            }
+
+            @Override
+            public int read(byte[] b) throws IOException {
+                return read(b, 0, b.length);
+            }
+
+            @Override
+            public int read(byte[] b, int off, int len) throws IOException {
+                int n = super.read(b, off, len);
+
+                if (n == -1 && !isDeleted) {
+                    super.close();
+                    isDeleted = tmpFile.delete();
                 }
-            };
+
+                return n;
+            }
+
+            @Override
+            public void close() throws IOException {
+                if (!isDeleted) {
+                    super.close();
+                    isDeleted = tmpFile.delete();
+                }
+            }
         }
     }
 }
\ No newline at end of file

Modified: chemistry/opencmis/trunk/chemistry-opencmis-test/chemistry-opencmis-test-fit/src/test/java/org/apache/chemistry/opencmis/fit/runtime/AbstractWriteObjectIT.java
URL: http://svn.apache.org/viewvc/chemistry/opencmis/trunk/chemistry-opencmis-test/chemistry-opencmis-test-fit/src/test/java/org/apache/chemistry/opencmis/fit/runtime/AbstractWriteObjectIT.java?rev=1204024&r1=1204023&r2=1204024&view=diff
==============================================================================
--- chemistry/opencmis/trunk/chemistry-opencmis-test/chemistry-opencmis-test-fit/src/test/java/org/apache/chemistry/opencmis/fit/runtime/AbstractWriteObjectIT.java
(original)
+++ chemistry/opencmis/trunk/chemistry-opencmis-test/chemistry-opencmis-test-fit/src/test/java/org/apache/chemistry/opencmis/fit/runtime/AbstractWriteObjectIT.java
Sat Nov 19 17:25:49 2011
@@ -71,8 +71,8 @@ public abstract class AbstractWriteObjec
 
         byte[] buf1 = content1.getBytes("UTF-8");
         ByteArrayInputStream in1 = new ByteArrayInputStream(buf1);
-        ContentStream contentStream = session.getObjectFactory().createContentStream(filename,
buf1.length,
-                mimetype, in1);
+        ContentStream contentStream = session.getObjectFactory().createContentStream(filename,
buf1.length, mimetype,
+                in1);
         assertNotNull(contentStream);
 
         ObjectId id = session.createDocument(properties, parentId, contentStream, VersioningState.NONE);
@@ -89,8 +89,54 @@ public abstract class AbstractWriteObjec
     }
 
     @Test
-    @Ignore
-    public void createHugeDocument() {
+    public void createBigDocument() {
+        ObjectId parentId = session.createObjectId(fixture.getTestRootId());
+        String folderName = UUID.randomUUID().toString();
+        String typeId = FixtureData.DOCUMENT_TYPE_ID.value();
+
+        Map<String, Object> properties = new HashMap<String, Object>();
+        properties.put(PropertyIds.NAME, folderName);
+        properties.put(PropertyIds.OBJECT_TYPE_ID, typeId);
+
+        String filename = UUID.randomUUID().toString();
+        String mimetype = "application/octet-stream";
+
+        final int size = 10 * 1024 * 1024; // 10 MiB
+
+        InputStream in = new InputStream() {
+
+            private int counter = -1;
+
+            @Override
+            public int read() throws IOException {
+                counter++;
+                if (counter >= size) {
+                    return -1;
+                }
+
+                return '0' + (counter / 10);
+            }
+        };
+
+        ContentStream contentStream = session.getObjectFactory().createContentStream(filename,
size, mimetype, in);
+        assertNotNull(contentStream);
+
+        long start = System.currentTimeMillis();
+        ObjectId id = session.createDocument(properties, parentId, contentStream, VersioningState.NONE);
+        long end = System.currentTimeMillis();
+
+        assertNotNull(id);
+        
+        Document doc = (Document) session.getObject(id);
+        assertNotNull(doc);
+        
+        doc.delete(true);
+
+        log.info("createDocument with " + size + " bytes:" + (end - start) + "ms");
+    }
+
+    @Test
+    public void setBigContent() {
         ObjectId parentId = session.createObjectId(fixture.getTestRootId());
         String folderName = UUID.randomUUID().toString();
         String typeId = FixtureData.DOCUMENT_TYPE_ID.value();
@@ -109,7 +155,7 @@ public abstract class AbstractWriteObjec
         Document doc = (Document) session.getObject(id);
         assertNotNull(doc);
 
-        final int size = 1 * 1024 * 1024 * 1024; // 1GB
+        final int size = 10 * 1024 * 1024; // 10 MiB
 
         InputStream in = new InputStream() {
 
@@ -133,7 +179,9 @@ public abstract class AbstractWriteObjec
         doc.setContentStream(contentStream, true);
         long end = System.currentTimeMillis();
 
-        log.info("setContentStream of " + size + " bytes:" + (end - start) + "ms");
+        doc.delete(true);
+
+        log.info("setContentStream with " + size + " bytes:" + (end - start) + "ms");
     }
 
     @Test
@@ -283,8 +331,8 @@ public abstract class AbstractWriteObjec
 
         byte[] buf1 = content1.getBytes("UTF-8");
         ByteArrayInputStream in1 = new ByteArrayInputStream(buf1);
-        ContentStream contentStream = session.getObjectFactory().createContentStream(filename,
buf1.length,
-                mimetype, in1);
+        ContentStream contentStream = session.getObjectFactory().createContentStream(filename,
buf1.length, mimetype,
+                in1);
         assertNotNull(contentStream);
 
         ObjectId id = session.createDocument(properties, parentId, null, VersioningState.NONE);
@@ -320,8 +368,8 @@ public abstract class AbstractWriteObjec
 
         byte[] buf1 = content1.getBytes("UTF-8");
         ByteArrayInputStream in1 = new ByteArrayInputStream(buf1);
-        ContentStream contentStream = session.getObjectFactory().createContentStream(filename,
buf1.length,
-                mimetype, in1);
+        ContentStream contentStream = session.getObjectFactory().createContentStream(filename,
buf1.length, mimetype,
+                in1);
         assertNotNull(contentStream);
 
         ObjectId id = session.createDocument(properties, parentId, null, VersioningState.NONE);
@@ -356,8 +404,8 @@ public abstract class AbstractWriteObjec
 
         byte[] buf1 = content1.getBytes("UTF-8");
         ByteArrayInputStream in1 = new ByteArrayInputStream(buf1);
-        ContentStream contentStream1 = session.getObjectFactory().createContentStream(filename1,
buf1.length,
-                mimetype, in1);
+        ContentStream contentStream1 = session.getObjectFactory().createContentStream(filename1,
buf1.length, mimetype,
+                in1);
         assertNotNull(contentStream1);
 
         ObjectId id = session.createDocument(properties, parentId, contentStream1, VersioningState.NONE);
@@ -369,8 +417,8 @@ public abstract class AbstractWriteObjec
 
         byte[] buf2 = content2.getBytes("UTF-8");
         ByteArrayInputStream in2 = new ByteArrayInputStream(buf2);
-        ContentStream contentStream2 = session.getObjectFactory().createContentStream(filename2,
buf2.length,
-                mimetype, in2);
+        ContentStream contentStream2 = session.getObjectFactory().createContentStream(filename2,
buf2.length, mimetype,
+                in2);
         assertNotNull(contentStream2);
 
         Document doc = (Document) session.getObject(id);
@@ -402,8 +450,8 @@ public abstract class AbstractWriteObjec
 
         byte[] buf1 = content1.getBytes("UTF-8");
         ByteArrayInputStream in1 = new ByteArrayInputStream(buf1);
-        ContentStream contentStream1 = session.getObjectFactory().createContentStream(filename1,
buf1.length,
-                mimetype, in1);
+        ContentStream contentStream1 = session.getObjectFactory().createContentStream(filename1,
buf1.length, mimetype,
+                in1);
         assertNotNull(contentStream1);
 
         ObjectId id = session.createDocument(properties, parentId, contentStream1, VersioningState.NONE);
@@ -415,8 +463,8 @@ public abstract class AbstractWriteObjec
 
         byte[] buf2 = content2.getBytes("UTF-8");
         ByteArrayInputStream in2 = new ByteArrayInputStream(buf2);
-        ContentStream contentStream2 = session.getObjectFactory().createContentStream(filename2,
buf2.length,
-                mimetype, in2);
+        ContentStream contentStream2 = session.getObjectFactory().createContentStream(filename2,
buf2.length, mimetype,
+                in2);
         assertNotNull(contentStream2);
 
         Document doc = (Document) session.getObject(id);



Mime
View raw message