commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject [01/10] commons-compress git commit: COMPRESS-382 first draft of preventing OOM in LZMA
Date Tue, 25 Apr 2017 18:46:01 GMT
Repository: commons-compress
Updated Branches:
  refs/heads/master 08f0aba10 -> 5def510d0


COMPRESS-382 first draft of preventing OOM in LZMA


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/70594bca
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/70594bca
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/70594bca

Branch: refs/heads/master
Commit: 70594bcaff1d553cbd35312bb63a33bcba0680d3
Parents: 0811386
Author: tballison <tallison@mitre.org>
Authored: Fri Apr 14 15:04:20 2017 -0400
Committer: tballison <tallison@mitre.org>
Committed: Fri Apr 14 15:04:20 2017 -0400

----------------------------------------------------------------------
 .../compressors/CompressorStreamFactory.java    |  15 ++++++++++--
 .../lzma/LZMACompressorInputStream.java         |  14 +++++++++--
 .../compressors/DetectCompressorTestCase.java   |  24 +++++++++++++++----
 src/test/resources/COMPRESS-382                 | Bin 0 -> 19 bytes
 4 files changed, 45 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/70594bca/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index 55da36a..d569a8a 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -349,6 +349,8 @@ public class CompressorStreamFactory implements CompressorStreamProvider
{
      */
     private volatile boolean decompressConcatenated = false;
 
+    private volatile int lzmaMemoryLimitKb = -1;
+
     /**
      * Create an instance with the decompress Concatenated option set to false.
      */
@@ -431,7 +433,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider
{
             }
 
             if (LZMAUtils.matches(signature, signatureLength) && LZMAUtils.isLZMACompressionAvailable())
{
-                return new LZMACompressorInputStream(in);
+                return new LZMACompressorInputStream(in, lzmaMemoryLimitKb);
             }
 
             if (FramedLZ4CompressorInputStream.matches(signature, signatureLength)) {
@@ -666,5 +668,14 @@ public class CompressorStreamFactory implements CompressorStreamProvider
{
         }
         this.decompressConcatenated = decompressConcatenated;
     }
-    
+
+    /**
+     * Set the maximum calculated memory usage for LZMA
+     * in KB.
+     *
+     * @param lzmaMemoryLimitKb
+     */
+    public void setLzmaMemoryLimitKb(int lzmaMemoryLimitKb) {
+        this.lzmaMemoryLimitKb = lzmaMemoryLimitKb;
+    }
 }

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/70594bca/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
index 556c78c..ea8a498 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
@@ -31,21 +31,31 @@ import org.apache.commons.compress.compressors.CompressorInputStream;
 public class LZMACompressorInputStream extends CompressorInputStream {
     private final InputStream in;
 
+    public LZMACompressorInputStream(final InputStream inputStream)
+            throws IOException {
+        in = new LZMAInputStream(inputStream, -1);
+    }
+
     /**
      * Creates a new input stream that decompresses LZMA-compressed data
      * from the specified input stream.
      *
      * @param       inputStream where to read the compressed data
      *
+     * @param       memoryLimitKb calculated memory use threshold.  Throws MemoryLimitException
+     *                            if calculate memory use is above this threshold
+     *
      * @throws      IOException if the input is not in the .lzma format,
      *                          the input is corrupt or truncated, the .lzma
      *                          headers specify sizes that are not supported
      *                          by this implementation, or the underlying
      *                          <code>inputStream</code> throws an exception
+     *
+     * @since 1.14
      */
-    public LZMACompressorInputStream(final InputStream inputStream)
+    public LZMACompressorInputStream(final InputStream inputStream, int memoryLimitKb)
             throws IOException {
-        in = new LZMAInputStream(inputStream);
+        in = new LZMAInputStream(inputStream, memoryLimitKb);
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/70594bca/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
index 674a2a0..9c446fa 100644
--- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
+++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
@@ -19,22 +19,25 @@
 package org.apache.commons.compress.compressors;
 
 import static org.apache.commons.compress.AbstractTestCase.getFile;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 
-import org.apache.commons.compress.compressors.CompressorException;
-import org.apache.commons.compress.compressors.CompressorInputStream;
-import org.apache.commons.compress.compressors.CompressorStreamFactory;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.apache.commons.compress.compressors.pack200.Pack200CompressorInputStream;
 import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
 import org.junit.Test;
+import org.tukaani.xz.MemoryLimitException;
 
 @SuppressWarnings("deprecation") // deliberately tests setDecompressConcatenated
 public final class DetectCompressorTestCase {
@@ -160,6 +163,19 @@ public final class DetectCompressorTestCase {
         }
     }
 
+    @Test
+    public void testLZMAMemoryLimit() throws Exception {
+        CompressorStreamFactory compressorStreamFactory = new CompressorStreamFactory();
+        compressorStreamFactory.setLzmaMemoryLimitKb(500);
+        try {
+            InputStream is = compressorStreamFactory.createCompressorInputStream(
+                    new BufferedInputStream(new FileInputStream(getFile("COMPRESS-382"))));
+            fail("Should have thrown memory limit exception");
+        } catch (CompressorException e) {
+            assertTrue(e.getCause() instanceof MemoryLimitException);
+        }
+    }
+
     private CompressorInputStream getStreamFor(final String resource)
             throws CompressorException, IOException {
         return factory.createCompressorInputStream(

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/70594bca/src/test/resources/COMPRESS-382
----------------------------------------------------------------------
diff --git a/src/test/resources/COMPRESS-382 b/src/test/resources/COMPRESS-382
new file mode 100644
index 0000000..be257f2
Binary files /dev/null and b/src/test/resources/COMPRESS-382 differ


Mime
View raw message