parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jul...@apache.org
Subject parquet-mr git commit: PARQUET-685 - Deprecated ParquetInputSplit constructor passes paramet…
Date Wed, 05 Oct 2016 20:21:44 GMT
Repository: parquet-mr
Updated Branches:
  refs/heads/master 07a42d3ff -> e6da0f682


PARQUET-685 - Deprecated ParquetInputSplit constructor passes paramet…

The problem was not discovered because the test was bugous. Updated both sides.

Author: Gabor Szadovszky <gabor.szadovszky@cloudera.com>

Closes #372 from gszadovszky/PARQUET-685 and squashes the following commits:

9cbeee2 [Gabor Szadovszky] PARQUET-685 - Deprecated ParquetInputSplit constructor passes parameters
in the wrong order.


Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/e6da0f68
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/e6da0f68
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/e6da0f68

Branch: refs/heads/master
Commit: e6da0f682436e1387ad68e86edf7418c0f7cb368
Parents: 07a42d3
Author: Gabor Szadovszky <gabor.szadovszky@cloudera.com>
Authored: Wed Oct 5 13:21:40 2016 -0700
Committer: Julien Le Dem <julien@dremio.com>
Committed: Wed Oct 5 13:21:40 2016 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/parquet/hadoop/ParquetInputSplit.java  | 3 +--
 .../test/java/org/apache/parquet/hadoop/TestInputFormat.java    | 5 +----
 2 files changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/e6da0f68/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputSplit.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputSplit.java
b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputSplit.java
index 8a13a3c..b97daa5 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputSplit.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputSplit.java
@@ -26,7 +26,6 @@ import java.io.DataOutput;
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.zip.GZIPInputStream;
@@ -89,7 +88,7 @@ public class ParquetInputSplit extends FileSplit implements Writable {
       String fileSchema,
       Map<String, String> extraMetadata,
       Map<String, String> readSupportMetadata) {
-    this(path, start, length, end(blocks, requestedSchema), hosts, offsets(blocks));
+    this(path, start, end(blocks, requestedSchema), length, hosts, offsets(blocks));
   }
 
   private static long end(List<BlockMetaData> blocks, String requestedSchema) {

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/e6da0f68/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java
index 9fe3008..d8b3de3 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java
@@ -481,11 +481,8 @@ public class TestInputFormat {
         .generateSplitInfo(blocks, hdfsBlocks, min, max);
 
     for (ClientSideMetadataSplitStrategy.SplitInfo splitInfo : splitInfos) {
-      BlockMetaData lastRowGroup = splitInfo.getRowGroups().get(splitInfo.getRowGroupCount()
- 1);
-      long end = lastRowGroup.getStartingPos() + lastRowGroup.getTotalByteSize();
-
       ParquetInputSplit split = new ParquetInputSplit(fileStatus.getPath(),
-          splitInfo.hdfsBlock.getOffset(), end, splitInfo.hdfsBlock.getHosts(),
+          splitInfo.hdfsBlock.getOffset(), splitInfo.hdfsBlock.getLength(), splitInfo.hdfsBlock.getHosts(),
           splitInfo.rowGroups, schema.toString(), null, null, extramd);
       splits.add(split);
     }


Mime
View raw message