drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ve...@apache.org
Subject [6/7] drill git commit: DRILL-4410: ListVector should initialize bits in allocateNew
Date Wed, 02 Mar 2016 23:52:25 GMT
DRILL-4410: ListVector should initialize bits in allocateNew

Remove large arrays.json files and generate it in /tmp, Check test results
Add another unit test for ListVector in TestValueVector, and use tempDir in TestComplexTypeReader

Closes #380


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/ae46d268
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/ae46d268
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/ae46d268

Branch: refs/heads/master
Commit: ae46d268b318b43f178250e34fc58028c016fc70
Parents: c95b543
Author: Minji Kim <minji@dremio.com>
Authored: Wed Feb 17 22:29:11 2016 -0800
Committer: vkorukanti <venki.korukanti@gmail.com>
Committed: Wed Mar 2 14:08:51 2016 -0800

----------------------------------------------------------------------
 .../exec/record/vector/TestValueVector.java     | 23 +++++++-
 .../complex/writer/TestComplexTypeReader.java   | 58 ++++++++++++++++++++
 .../drill/exec/vector/complex/ListVector.java   |  1 +
 3 files changed, 81 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/ae46d268/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
index f6bcf97..f6044e1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
@@ -25,7 +25,7 @@ import java.nio.charset.Charset;
 
 import org.apache.drill.common.AutoCloseables;
 import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.MinorType;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.ExecTest;
@@ -47,6 +47,7 @@ import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.memory.RootAllocatorFactory;
 import org.apache.drill.exec.proto.UserBitShared;
 import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.vector.BaseValueVector;
 import org.apache.drill.exec.vector.BitVector;
 import org.apache.drill.exec.vector.NullableFloat4Vector;
@@ -56,6 +57,7 @@ import org.apache.drill.exec.vector.RepeatedIntVector;
 import org.apache.drill.exec.vector.UInt4Vector;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.VarCharVector;
+import org.apache.drill.exec.vector.complex.ListVector;
 import org.apache.drill.exec.vector.complex.MapVector;
 import org.apache.drill.exec.vector.complex.RepeatedListVector;
 import org.apache.drill.exec.vector.complex.RepeatedMapVector;
@@ -789,4 +791,23 @@ the interface to load has changed
       }
     });
   }
+
+  @Test
+  public void testListVectorShouldNotThrowOversizedAllocationException() throws Exception
{
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH,
+            Types.optional(TypeProtos.MinorType.LIST));
+    ListVector vector = new ListVector(field, allocator, null);
+    ListVector vectorFrom = new ListVector(field, allocator, null);
+    vectorFrom.allocateNew();
+
+    for (int i = 0; i < 10000; i++) {
+      vector.allocateNew();
+      vector.copyFromSafe(0, 0, vectorFrom);
+      vector.clear();
+    }
+
+    vectorFrom.clear();
+    vector.clear();
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/ae46d268/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
index 077a3ca..c3cfb68 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
@@ -19,10 +19,21 @@
 package org.apache.drill.exec.vector.complex.writer;
 
 import org.apache.drill.BaseTestQuery;
+import org.apache.drill.TestBuilder;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
 
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static org.apache.drill.TestBuilder.listOf;
+
 public class TestComplexTypeReader extends BaseTestQuery{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestComplexTypeReader.class);
 
@@ -241,4 +252,51 @@ public class TestComplexTypeReader extends BaseTestQuery{
             .go();
   }
 
+  @Test  // DRILL-4410
+  // ListVector allocation
+  public void test_array() throws Exception{
+
+    long numRecords = 100000;
+
+    String tempDir = BaseTestQuery.getTempDir("ComplexTypeWriter");
+    String file1 = tempDir + TestComplexTypeReader.class.getName() + "arrays1.json";
+    String file2 = tempDir + TestComplexTypeReader.class.getName() + "arrays2.json";
+    Path path1 = Paths.get(file1);
+    Path path2 = Paths.get(file2);
+
+    String arrayString = "[ \"abcdef\", \"ghijkl\", \"mnopqr\", \"stuvwx\", \"yz1234\", \"567890\"
] ";
+    Files.deleteIfExists(path1);
+    Files.deleteIfExists(path2);
+    Files.createFile(path1);
+    Files.createFile(path2);
+
+    try(PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file1, true))))
{
+      for (long i = 0; i < numRecords; i++) {
+        out.println("{ \"id\" : " + i + ", \"array\" : " + arrayString + "}");
+      }
+    }catch (IOException e) {
+      throw e;
+    }
+
+    try(PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file2, true))))
{
+      for (long i = 0; i < numRecords; i++) {
+        out.println("{ \"id\" : " + i + ", \"array\" : " + arrayString + "}");
+      }
+    }catch (IOException e) {
+      throw e;
+    }
+
+    String queryString = "select * from dfs.`" + file1 + "` `arrays1` INNER JOIN dfs.`" +
file2 + "` `arrays2` ON "
+            + "(`arrays1`.id = `arrays2`.id)";
+    TestBuilder testBuilder = testBuilder().sqlQuery(queryString).unOrdered();
+    testBuilder.baselineColumns("id", "id0", "array", "array0");
+    for (long i = 0; i < numRecords; i++) {
+      testBuilder.baselineValues(i, i, listOf("abcdef", "ghijkl", "mnopqr", "stuvwx", "yz1234",
"567890"),
+              listOf("abcdef", "ghijkl", "mnopqr", "stuvwx", "yz1234", "567890"));
+    }
+    testBuilder.go();
+
+    Files.deleteIfExists(path1);
+    Files.deleteIfExists(path2);
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/ae46d268/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
index ba30241..33d6ddc 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
@@ -70,6 +70,7 @@ public class ListVector extends BaseRepeatedValueVector {
   @Override
   public void allocateNew() throws OutOfMemoryException {
     super.allocateNewSafe();
+    bits.allocateNewSafe();
   }
 
   public void transferTo(ListVector target) {


Mime
View raw message