Repository: incubator-parquet-mr
Updated Branches:
refs/heads/master 031a762d1 -> b58789c5b
PARQUET-180: Update use of TBinaryProtocol#setReadLength.
This is no longer supported in thrift 0.9.2 and was only used
defensively. The reason to remove it now is to avoid linker errors when
the wrong version of thrift is found in the classpath.
Author: Ryan Blue <blue@apache.org>
Closes #118 from rdblue/PARQUET-180 and squashes the following commits:
5100424 [Ryan Blue] PARQUET-180: Dynamic use of TBinaryProtocol#setReadLength.
Project: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/commit/b58789c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/tree/b58789c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/diff/b58789c5
Branch: refs/heads/master
Commit: b58789c5badc4f9680ec5724568af05a84670e22
Parents: 031a762
Author: Ryan Blue <blue@apache.org>
Authored: Wed Mar 11 15:21:45 2015 -0700
Committer: Ryan Blue <blue@apache.org>
Committed: Wed Mar 11 15:21:45 2015 -0700
----------------------------------------------------------------------
.../parquet/hadoop/thrift/ThriftBytesWriteSupport.java | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/b58789c5/parquet-thrift/src/main/java/parquet/hadoop/thrift/ThriftBytesWriteSupport.java
----------------------------------------------------------------------
diff --git a/parquet-thrift/src/main/java/parquet/hadoop/thrift/ThriftBytesWriteSupport.java
b/parquet-thrift/src/main/java/parquet/hadoop/thrift/ThriftBytesWriteSupport.java
index d585e71..79a03aa 100644
--- a/parquet-thrift/src/main/java/parquet/hadoop/thrift/ThriftBytesWriteSupport.java
+++ b/parquet-thrift/src/main/java/parquet/hadoop/thrift/ThriftBytesWriteSupport.java
@@ -119,6 +119,16 @@ public class ThriftBytesWriteSupport extends WriteSupport<BytesWritable>
{
return thriftWriteSupport.init(configuration);
}
+ private static boolean IS_READ_LENGTH_SETABLE = false;
+ static {
+ try {
+ TBinaryProtocol.class.getMethod("setReadLength", int.class);
+ IS_READ_LENGTH_SETABLE = true;
+ } catch (NoSuchMethodException e) {
+ IS_READ_LENGTH_SETABLE = false;
+ }
+ }
+
private TProtocol protocol(BytesWritable record) {
TProtocol protocol = protocolFactory.getProtocol(new TIOStreamTransport(new ByteArrayInputStream(record.getBytes())));
@@ -126,9 +136,10 @@ public class ThriftBytesWriteSupport extends WriteSupport<BytesWritable>
{
so if the data is corrupted, it could read a big integer as the length of the binary
and therefore causes OOM to happen.
Currently this fix only applies to TBinaryProtocol which has the setReadLength defined.
*/
- if (protocol instanceof TBinaryProtocol) {
+ if (IS_READ_LENGTH_SETABLE && protocol instanceof TBinaryProtocol) {
((TBinaryProtocol)protocol).setReadLength(record.getLength());
}
+
return protocol;
}
|