commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject [07/10] commons-compress git commit: COMPRESS-382 and COMPRESS-386 -- hard reset to expunge static parameter proposal. Updated MemoryLimitException to include cause when it exists. Updated LZWInputStream to calculate estimated memory on maxTableSize*6 t
Date Tue, 25 Apr 2017 18:46:07 GMT
COMPRESS-382 and COMPRESS-386 -- hard reset to expunge static parameter
proposal.  Updated MemoryLimitException to include cause when it exists.
Updated LZWInputStream to calculate estimated memory on maxTableSize*6
to account for int[] and byte[], byte[].


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/4e442bb4
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/4e442bb4
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/4e442bb4

Branch: refs/heads/master
Commit: 4e442bb44280243f6e0e72d845fa8275d665ca84
Parents: b10528a
Author: tballison <tallison@mitre.org>
Authored: Tue Apr 25 08:44:21 2017 -0400
Committer: tballison <tallison@mitre.org>
Committed: Tue Apr 25 08:44:21 2017 -0400

----------------------------------------------------------------------
 .../commons/compress/MemoryLimitException.java  | 30 +++++++++++++++++---
 .../lzma/LZMACompressorInputStream.java         |  2 +-
 .../compressors/lzw/LZWInputStream.java         | 20 +++++++++----
 .../compressors/xz/XZCompressorInputStream.java |  8 +++---
 .../compressors/z/ZCompressorInputStream.java   |  2 +-
 .../compressors/DetectCompressorTestCase.java   |  2 +-
 6 files changed, 47 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/MemoryLimitException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/MemoryLimitException.java b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
index 8922ed2..122edb1 100644
--- a/src/main/java/org/apache/commons/compress/MemoryLimitException.java
+++ b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
@@ -30,11 +30,33 @@ import java.io.IOException;
  */
 public class MemoryLimitException extends IOException {
 
-    public MemoryLimitException(String message) {
-        super(message);
+    //long instead of int to account for overflow for corrupt files
+    private final long memoryNeededInKb;
+    private final int memoryLimitInKb;
+
+    public MemoryLimitException(long memoryNeededInKb, int memoryLimitInKb) {
+        super(buildMessage(memoryNeededInKb, memoryLimitInKb));
+        this.memoryNeededInKb = memoryNeededInKb;
+        this.memoryLimitInKb = memoryLimitInKb;
+    }
+
+    public MemoryLimitException(long memoryNeededInKb, int memoryLimitInKb, Exception e)
{
+        super(buildMessage(memoryNeededInKb, memoryLimitInKb), e);
+        this.memoryNeededInKb = memoryNeededInKb;
+        this.memoryLimitInKb = memoryLimitInKb;
+    }
+
+    public long getMemoryNeededInKb() {
+        return memoryNeededInKb;
+    }
+
+    public int getMemoryLimitInKb() {
+        return memoryLimitInKb;
     }
 
-    public MemoryLimitException(String message, Exception e) {
-        super(message, e);
+    private static String buildMessage(long memoryNeededInKb, int memoryLimitInKb) {
+        return "" + memoryNeededInKb + " kb of memory would be needed; limit was "
+                + memoryLimitInKb + " kb. " +
+                "If the file is not corrupt, consider increasing the memory limit.";
     }
 }

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
index 7782be8..5723b5c 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
@@ -61,7 +61,7 @@ public class LZMACompressorInputStream extends CompressorInputStream {
             in = new LZMAInputStream(inputStream, memoryLimitInKb);
         } catch (org.tukaani.xz.MemoryLimitException e) {
             //convert to commons-compress exception
-            throw new MemoryLimitException("exceeded calculated memory limit", e);
+            throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
index 350b4b0..2a6127d 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
@@ -114,16 +114,24 @@ public abstract class LZWInputStream extends CompressorInputStream {
 
     /**
      * Initializes the arrays based on the maximum code size.
+     * First checks that the estimated memory usage is below memoryLimitInKb
+     *
      * @param maxCodeSize maximum code size
-     * @param memoryLimitInKb maximum allowed table size in Kb
-     * @throws MemoryLimitException if maxTableSize is > memoryLimitInKb
+     * @param memoryLimitInKb maximum allowed estimated memory usage in Kb
+     * @throws MemoryLimitException if estimated memory usage is greater than memoryLimitInKb
      */
     protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb)
             throws MemoryLimitException {
-        final int maxTableSize = 1 << maxCodeSize;
-        if (memoryLimitInKb > -1 && maxTableSize > memoryLimitInKb*1024) {
-            throw new MemoryLimitException("Tried to allocate "+maxTableSize +
-                    " but memoryLimitInKb only allows "+(memoryLimitInKb*1024));
+
+        if (memoryLimitInKb > -1) {
+            final int maxTableSize = 1 << maxCodeSize;
+            //account for potential overflow
+            long memoryUsageInBytes = (long) maxTableSize * 6;//(4 (prefixes) + 1 (characters)
+1 (outputStack))
+            long memoryUsageInKb = memoryUsageInBytes >> 10;
+
+            if (memoryUsageInKb > (long)memoryLimitInKb) {
+                throw new MemoryLimitException(memoryUsageInKb, memoryLimitInKb);
+            }
         }
         initializeTables(maxCodeSize);
     }

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
index b378212..fc0cbf5 100644
--- a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
@@ -120,7 +120,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
      * @since 1.14
      */
     public XZCompressorInputStream(InputStream inputStream,
-                                   boolean decompressConcatenated, int memoryLimitInKb)
+                                   boolean decompressConcatenated, final int memoryLimitInKb)
             throws IOException {
         if (decompressConcatenated) {
             in = new XZInputStream(inputStream, memoryLimitInKb);
@@ -136,7 +136,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
             count(ret == -1 ? -1 : 1);
             return ret;
         } catch (org.tukaani.xz.MemoryLimitException e) {
-            throw new MemoryLimitException("Exceeded memory limit", e);
+            throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e);
         }
     }
 
@@ -148,7 +148,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
             return ret;
         } catch (org.tukaani.xz.MemoryLimitException e) {
             //convert to commons-compress MemoryLimtException
-            throw new MemoryLimitException("Exceeded memory limit", e);
+            throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e);
         }
     }
 
@@ -158,7 +158,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
             return in.skip(n);
         } catch (org.tukaani.xz.MemoryLimitException e) {
             //convert to commons-compress MemoryLimtException
-            throw new MemoryLimitException("Excedded memory limit", e);
+            throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
index 64387e3..994e102 100644
--- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
@@ -38,7 +38,7 @@ public class ZCompressorInputStream extends LZWInputStream {
     private final int maxCodeSize;
     private long totalCodesRead = 0;
 
-    public ZCompressorInputStream(final InputStream inputStream, int memoryLimitInKb)
+    public ZCompressorInputStream(final InputStream inputStream, final int memoryLimitInKb)
             throws IOException {
         super(inputStream, ByteOrder.LITTLE_ENDIAN);
         final int firstByte = (int) in.readBits(8);

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
index b70d3c7..844b168 100644
--- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
+++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java
@@ -210,7 +210,7 @@ public final class DetectCompressorTestCase {
             return fac.createCompressorInputStream(is);
         } catch (CompressorException e) {
             if (e.getCause() != null && e.getCause() instanceof Exception) {
-                //unwrap cause to reveal MemoryLimiteException
+                //unwrap cause to reveal MemoryLimitException
                 throw (Exception)e.getCause();
             } else {
                 throw e;


Mime
View raw message