parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jul...@apache.org
Subject parquet-mr git commit: PARQUET-623: Fix DeltaByteArrayReader#skip.
Date Thu, 08 Sep 2016 21:22:37 GMT
Repository: parquet-mr
Updated Branches:
  refs/heads/master 6dad1e3bd -> 044de16c1


PARQUET-623: Fix DeltaByteArrayReader#skip.

Previously, this passed the skip to the underlying readers, but would
not update previous and would corrupt values or cause exceptions.

Author: Ryan Blue <blue@apache.org>

Closes #366 from rdblue/PARQUET-623-fix-delta-byte-array-skip and squashes the following commits:

f85800c [Ryan Blue] PARQUET-623: Fix DeltaByteArrayReader#skip.


Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/044de16c
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/044de16c
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/044de16c

Branch: refs/heads/master
Commit: 044de16c14076019f87763b7b58c45664ee57c11
Parents: 6dad1e3
Author: Ryan Blue <blue@apache.org>
Authored: Thu Sep 8 14:22:30 2016 -0700
Committer: Julien Le Dem <julien@dremio.com>
Committed: Thu Sep 8 14:22:30 2016 -0700

----------------------------------------------------------------------
 .../values/deltastrings/DeltaByteArrayReader.java |  4 ++--
 .../values/deltastrings/TestDeltaByteArray.java   | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/044de16c/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
b/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
index c2cfd6d..742b515 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/deltastrings/DeltaByteArrayReader.java
@@ -55,8 +55,8 @@ public class DeltaByteArrayReader extends ValuesReader implements RequiresPrevio
   
   @Override
   public void skip() {
-    prefixLengthReader.skip();
-    suffixReader.skip();
+    // read the next value to skip so that previous is correct.
+    readBytes();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/044de16c/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
----------------------------------------------------------------------
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
b/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
index 770f4dc..4f8f40c 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
@@ -19,6 +19,7 @@
 package org.apache.parquet.column.values.deltastrings;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 
 import org.junit.Test;
 import org.junit.Assert;
@@ -50,6 +51,13 @@ public class TestDeltaByteArray {
   }
 
   @Test
+  public void testRandomStringsWithSkip() throws Exception {
+    DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator());
+    DeltaByteArrayReader reader = new DeltaByteArrayReader();
+    assertReadWriteWithSkip(writer, reader, randvalues);
+  }
+
+  @Test
   public void testLengths() throws IOException {
     DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator());
     ValuesReader reader = new DeltaBinaryPackingValuesReader();
@@ -81,6 +89,16 @@ public class TestDeltaByteArray {
     }
   }
 
+  private void assertReadWriteWithSkip(DeltaByteArrayWriter writer, DeltaByteArrayReader
reader, String[] vals) throws Exception {
+    Utils.writeData(writer, vals);
+
+    reader.initFromPage(vals.length, writer.getBytes().toByteBuffer(), 0);
+    for (int i = 0; i < vals.length; i += 2) {
+      Assert.assertEquals(Binary.fromString(vals[i]), reader.readBytes());
+      reader.skip();
+    }
+  }
+
   @Test
   public void testWriterReset() throws Exception {
     DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator());


Mime
View raw message