parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jul...@apache.org
Subject parquet-mr git commit: PARQUET-340: MemoryManager: max memory can be truncated
Date Mon, 20 Jul 2015 16:59:42 GMT
Repository: parquet-mr
Updated Branches:
  refs/heads/master 1dd5cec08 -> 83406b73e


PARQUET-340: MemoryManager: max memory can be truncated

Using float will cause the max heap limit to be limited to 2147483647
due to math.round(float) if used with a large heap. This should be a double
precision to prevent rounding to an int32 before storing into a long.

Author: Chris Bannister <c.bannister@gmail.com>

Closes #246 from Zariel/default-mem-truncated and squashes the following commits:

bf375f6 [Chris Bannister] MemoryManager: ensure max memory is not truncated


Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/83406b73
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/83406b73
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/83406b73

Branch: refs/heads/master
Commit: 83406b73e70a251eec5daae34f1bd8d554cdddec
Parents: 1dd5cec
Author: Chris Bannister <c.bannister@gmail.com>
Authored: Mon Jul 20 09:59:29 2015 -0700
Committer: julien <julien@twitter.com>
Committed: Mon Jul 20 09:59:29 2015 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/parquet/hadoop/MemoryManager.java     | 2 +-
 .../src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/83406b73/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java
index f997d08..0c56bb2 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java
@@ -57,7 +57,7 @@ public class MemoryManager {
 
     memoryPoolRatio = ratio;
     minMemoryAllocation = minAllocation;
-    totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax
+    totalMemoryPool = Math.round((double) ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax
         () * ratio);
     LOG.debug(String.format("Allocated total memory pool is: %,d", totalMemoryPool));
   }

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/83406b73/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java
b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java
index bbe7443..fb71b86 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java
@@ -54,7 +54,7 @@ public class TestMemoryManager {
   @Before
   public void setUp() {
     GroupWriteSupport.setSchema(MessageTypeParser.parseMessageType(writeSchema),conf);
-    expectPoolSize = Math.round(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax
+    expectPoolSize = Math.round((double) ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax
         () * MemoryManager.DEFAULT_MEMORY_POOL_RATIO);
     rowGroupSize = (int) Math.floor(expectPoolSize / 2);
     conf.setInt(ParquetOutputFormat.BLOCK_SIZE, rowGroupSize);


Mime
View raw message