parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject [24/50] [abbrv] parquet-mr git commit: PARQUET-623: Fix DeltaByteArrayReader#skip.
Date Thu, 19 Jan 2017 01:27:35 GMT
PARQUET-623: Fix DeltaByteArrayReader#skip.

Previously, this passed the skip to the underlying readers, but would
not update previous and would corrupt values or cause exceptions.

Author: Ryan Blue <blue@apache.org>

Closes #366 from rdblue/PARQUET-623-fix-delta-byte-array-skip and squashes the following commits:

f85800c [Ryan Blue] PARQUET-623: Fix DeltaByteArrayReader#skip.


Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/6e95d450
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/6e95d450
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/6e95d450

Branch: refs/heads/parquet-1.8.x
Commit: 6e95d45009848a402aed665b3d92bd2ee6a87c3b
Parents: 71e43ad
Author: Ryan Blue <blue@apache.org>
Authored: Thu Sep 8 14:22:30 2016 -0700
Committer: Ryan Blue <blue@apache.org>
Committed: Mon Jan 9 16:54:54 2017 -0800

----------------------------------------------------------------------
 .../values/deltastrings/DeltaByteArrayReader.java |  4 ++--
 .../values/deltastrings/TestDeltaByteArray.java   | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/6e95d450/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
b/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
index 87ec08e..14fa1ee 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
@@ -54,8 +54,8 @@ public class DeltaByteArrayReader extends ValuesReader implements RequiresPrevio
 
   @Override
   public void skip() {
-    prefixLengthReader.skip();
-    suffixReader.skip();
+    // read the next value to skip so that previous is correct.
+    readBytes();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/6e95d450/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
b/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
index b62ef84..263fce4 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
@@ -19,6 +19,7 @@
 package org.apache.parquet.column.values.deltastrings;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 
 import org.junit.Test;
 import org.junit.Assert;
@@ -49,6 +50,13 @@ public class TestDeltaByteArray {
   }
 
   @Test
+  public void testRandomStringsWithSkip() throws Exception {
+    DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024);
+    DeltaByteArrayReader reader = new DeltaByteArrayReader();
+    assertReadWriteWithSkip(writer, reader, randvalues);
+  }
+
+  @Test
   public void testLengths() throws IOException {
     DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024);
     ValuesReader reader = new DeltaBinaryPackingValuesReader();
@@ -80,6 +88,16 @@ public class TestDeltaByteArray {
     }
   }
 
+  private void assertReadWriteWithSkip(DeltaByteArrayWriter writer, DeltaByteArrayReader
reader, String[] vals) throws Exception {
+    Utils.writeData(writer, vals);
+
+    reader.initFromPage(vals.length, writer.getBytes().toByteArray(), 0);
+    for (int i = 0; i < vals.length; i += 2) {
+      Assert.assertEquals(Binary.fromString(vals[i]), reader.readBytes());
+      reader.skip();
+    }
+  }
+
   @Test
   public void testWriterReset() throws Exception {
     DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024);


Mime
View raw message