parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject git commit: PARQUET-82: Check page size is valid when writing.
Date Tue, 23 Sep 2014 19:14:23 GMT
Repository: incubator-parquet-mr
Updated Branches:
  refs/heads/master 0eb963795 -> 59c58d0b8


PARQUET-82: Check page size is valid when writing.

Author: Ryan Blue <rblue@cloudera.com>

Closes #48 from rdblue/PARQUET-82-check-page-size and squashes the following commits:

9f31402 [Ryan Blue] PARQUET-82: Check page size is valid when writing.


Project: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/commit/59c58d0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/tree/59c58d0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/diff/59c58d0b

Branch: refs/heads/master
Commit: 59c58d0b829aa156f038cc900b803508f8849765
Parents: 0eb9637
Author: Ryan Blue <rblue@cloudera.com>
Authored: Tue Sep 23 12:14:17 2014 -0700
Committer: Ryan Blue <rblue@cloudera.com>
Committed: Tue Sep 23 12:14:17 2014 -0700

----------------------------------------------------------------------
 .../hadoop/ColumnChunkPageWriteStore.java       | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/59c58d0b/parquet-hadoop/src/main/java/parquet/hadoop/ColumnChunkPageWriteStore.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/parquet/hadoop/ColumnChunkPageWriteStore.java b/parquet-hadoop/src/main/java/parquet/hadoop/ColumnChunkPageWriteStore.java
index 279eb56..6d7f685 100644
--- a/parquet-hadoop/src/main/java/parquet/hadoop/ColumnChunkPageWriteStore.java
+++ b/parquet-hadoop/src/main/java/parquet/hadoop/ColumnChunkPageWriteStore.java
@@ -77,8 +77,18 @@ class ColumnChunkPageWriteStore implements PageWriteStore {
                           Encoding dlEncoding,
                           Encoding valuesEncoding) throws IOException {
       long uncompressedSize = bytes.size();
+      if (uncompressedSize > Integer.MAX_VALUE) {
+        throw new ParquetEncodingException(
+            "Cannot write page larger than Integer.MAX_VALUE bytes: " +
+                uncompressedSize);
+      }
       BytesInput compressedBytes = compressor.compress(bytes);
       long compressedSize = compressedBytes.size();
+      if (compressedSize > Integer.MAX_VALUE) {
+        throw new ParquetEncodingException(
+            "Cannot write compressed page larger than Integer.MAX_VALUE bytes: "
+                + compressedSize);
+      }
       BooleanStatistics statistics = new BooleanStatistics(); // dummy stats object
       parquetMetadataConverter.writeDataPageHeader(
           (int)uncompressedSize,
@@ -107,8 +117,18 @@ class ColumnChunkPageWriteStore implements PageWriteStore {
                           Encoding dlEncoding,
                           Encoding valuesEncoding) throws IOException {
       long uncompressedSize = bytes.size();
+      if (uncompressedSize > Integer.MAX_VALUE) {
+        throw new ParquetEncodingException(
+            "Cannot write page larger than Integer.MAX_VALUE bytes: " +
+            uncompressedSize);
+      }
       BytesInput compressedBytes = compressor.compress(bytes);
       long compressedSize = compressedBytes.size();
+      if (compressedSize > Integer.MAX_VALUE) {
+        throw new ParquetEncodingException(
+            "Cannot write compressed page larger than Integer.MAX_VALUE bytes: "
+            + compressedSize);
+      }
       parquetMetadataConverter.writeDataPageHeader(
           (int)uncompressedSize,
           (int)compressedSize,


Mime
View raw message