commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject [06/10] commons-compress git commit: COMPRESS-382 and COMPRESS-386 -- take 4, clean up and allow for overflow via longs.
Date Tue, 25 Apr 2017 18:46:06 GMT
COMPRESS-382 and COMPRESS-386 -- take 4, clean up and allow
for overflow via longs.


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/26eab98e
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/26eab98e
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/26eab98e

Branch: refs/heads/master
Commit: 26eab98e3a0d403587cc915eaa8f7d4b7b894cef
Parents: 0e8ff9c
Author: tballison <tallison@mitre.org>
Authored: Mon Apr 24 22:09:59 2017 -0400
Committer: tballison <tallison@mitre.org>
Committed: Mon Apr 24 22:09:59 2017 -0400

----------------------------------------------------------------------
 .../apache/commons/compress/MemoryLimit.java    | 15 +++++++++++
 .../commons/compress/MemoryLimitException.java  | 18 +++++++++-----
 .../lzma/LZMACompressorInputStream.java         |  2 +-
 .../compressors/lzw/LZWInputStream.java         | 26 ++++++++------------
 .../compressors/xz/XZCompressorInputStream.java |  6 ++---
 .../compressors/z/ZCompressorInputStream.java   |  3 +--
 6 files changed, 42 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/MemoryLimit.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/MemoryLimit.java b/src/main/java/org/apache/commons/compress/MemoryLimit.java
index e03e8e7..b43eb86 100644
--- a/src/main/java/org/apache/commons/compress/MemoryLimit.java
+++ b/src/main/java/org/apache/commons/compress/MemoryLimit.java
@@ -62,4 +62,19 @@ public class MemoryLimit {
     public static int getMemoryLimitInKb() {
         return MEMORY_LIMIT_IN_KB;
     }
+
+    public static void checkLimitInKb(long memoryNeeded) throws MemoryLimitException {
+        if (memoryNeeded < 0) {
+            throw new IllegalArgumentException("MemoryLimit must be > -1");
+        }
+
+        if (memoryNeeded >> 10 > Integer.MAX_VALUE) {
+            throw new MemoryLimitException(memoryNeeded,
+                    (MEMORY_LIMIT_IN_KB < 0) ? Integer.MAX_VALUE : MEMORY_LIMIT_IN_KB);
+        }
+
+        if (MEMORY_LIMIT_IN_KB > -1 && memoryNeeded > MEMORY_LIMIT_IN_KB) {
+                throw new MemoryLimitException(memoryNeeded, MEMORY_LIMIT_IN_KB);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/MemoryLimitException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/MemoryLimitException.java b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
index 8922ed2..a243a17 100644
--- a/src/main/java/org/apache/commons/compress/MemoryLimitException.java
+++ b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
@@ -26,15 +26,21 @@ import java.io.IOException;
  * if a stream tries to allocate a byte array that is larger than
  * the allowable limit.
  *
+ * <p/>
+ * Set the global memory limit via {@link MemoryLimit#setMemoryLimitInKb(int)}.
+ *
  * @since 1.14
  */
 public class MemoryLimitException extends IOException {
 
-    public MemoryLimitException(String message) {
-        super(message);
-    }
-
-    public MemoryLimitException(String message, Exception e) {
-        super(message, e);
+    /**
+     *
+     * @param memoryNeeded estimated memory needed
+     * @param memoryLimit memory limit applied
+     */
+    public MemoryLimitException(long memoryNeeded, int memoryLimit) {
+        super("" + memoryNeeded + " KiB of memory would be needed; limit was "
+                + memoryLimit + " KiB.  If the file is not corrupt, consider " +
+                "increasing MemoryLimit.MEMORY_LIMIT_IN_KB.");
     }
 }

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
index 6931541..35d7e12 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
@@ -56,7 +56,7 @@ public class LZMACompressorInputStream extends CompressorInputStream {
             in = new LZMAInputStream(inputStream, MemoryLimit.getMemoryLimitInKb());
         } catch (org.tukaani.xz.MemoryLimitException e) {
             //convert to commons-compress exception
-            throw new MemoryLimitException("exceeded calculated memory limit", e);
+            throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit());
         }
     }
 

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
index 350b4b0..ed2c476 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteOrder;
 
+import org.apache.commons.compress.MemoryLimit;
 import org.apache.commons.compress.MemoryLimitException;
 import org.apache.commons.compress.compressors.CompressorInputStream;
 import org.apache.commons.compress.utils.BitInputStream;
@@ -115,25 +116,18 @@ public abstract class LZWInputStream extends CompressorInputStream {
     /**
      * Initializes the arrays based on the maximum code size.
      * @param maxCodeSize maximum code size
-     * @param memoryLimitInKb maximum allowed table size in Kb
-     * @throws MemoryLimitException if maxTableSize is > memoryLimitInKb
+     *
+     * @throws MemoryLimitException
+     *      if the calculated memory usage, based on the maxTableSize,
+     *      is &gt; {@link MemoryLimit#MEMORY_LIMIT_IN_KB}
      */
-    protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb)
-            throws MemoryLimitException {
+    protected void initializeTables(final int maxCodeSize) throws MemoryLimitException {
         final int maxTableSize = 1 << maxCodeSize;
-        if (memoryLimitInKb > -1 && maxTableSize > memoryLimitInKb*1024) {
-            throw new MemoryLimitException("Tried to allocate "+maxTableSize +
-                    " but memoryLimitInKb only allows "+(memoryLimitInKb*1024));
-        }
-        initializeTables(maxCodeSize);
-    }
 
-    /**
-     * Initializes the arrays based on the maximum code size.
-     * @param maxCodeSize maximum code size
-     */
-    protected void initializeTables(final int maxCodeSize) {
-        final int maxTableSize = 1 << maxCodeSize;
+        //account for potential overflow
+        long memoryUsageInBytes = (long)maxTableSize * 6;//(4 (prefixes) + 1 (characters)
+1 (outputStack))
+        MemoryLimit.checkLimitInKb(memoryUsageInBytes >> 10);
+
         prefixes = new int[maxTableSize];
         characters = new byte[maxTableSize];
         outputStack = new byte[maxTableSize];

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
index 3f977ef..e80cbcf 100644
--- a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
@@ -110,7 +110,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
             count(ret == -1 ? -1 : 1);
             return ret;
         } catch (org.tukaani.xz.MemoryLimitException e) {
-            throw new MemoryLimitException("Exceeded memory limit", e);
+            throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit());
         }
     }
 
@@ -122,7 +122,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
             return ret;
         } catch (org.tukaani.xz.MemoryLimitException e) {
             //convert to commons-compress MemoryLimtException
-            throw new MemoryLimitException("Exceeded memory limit", e);
+            throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit());
         }
     }
 
@@ -132,7 +132,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
             return in.skip(n);
         } catch (org.tukaani.xz.MemoryLimitException e) {
             //convert to commons-compress MemoryLimtException
-            throw new MemoryLimitException("Excedded memory limit", e);
+            throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit());
         }
     }
 

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
index ca61cd3..28b69a2 100644
--- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
@@ -22,7 +22,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteOrder;
 
-import org.apache.commons.compress.MemoryLimit;
 import org.apache.commons.compress.compressors.lzw.LZWInputStream;
 
 /**
@@ -52,7 +51,7 @@ public class ZCompressorInputStream extends LZWInputStream {
         if (blockMode) {
             setClearCode(DEFAULT_CODE_SIZE);
         }
-        initializeTables(maxCodeSize, MemoryLimit.getMemoryLimitInKb());
+        initializeTables(maxCodeSize);
         clearEntries();
     }
     


Mime
View raw message