drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From par...@apache.org
Subject [3/4] drill git commit: DRILL-1660: return accurate buffers size when repeated map vector has no underlying vectors
Date Thu, 27 Nov 2014 17:32:04 GMT
DRILL-1660: return accurate buffers size when repeated map vector has no underlying vectors


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a94a028c
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a94a028c
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a94a028c

Branch: refs/heads/master
Commit: a94a028cfc37fc5b5e8eed319a389cc74682ba68
Parents: cf9e71d
Author: Hanifi Gunes <hgunes@maprtech.com>
Authored: Wed Nov 12 13:03:57 2014 -0800
Committer: Parth Chandra <pchandra@maprtech.com>
Committed: Wed Nov 26 14:54:25 2014 -0800

----------------------------------------------------------------------
 .../fn/TestJsonReaderWithSparseFiles.java       | 142 ++++++++++---------
 .../complex/fn/single-record-with-empties.json  |   1 +
 2 files changed, 78 insertions(+), 65 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/a94a028c/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
index 3cfdc1d..3765a29 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
@@ -22,11 +22,10 @@ import java.util.List;
 import java.util.Objects;
 
 import org.apache.drill.BaseTestQuery;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.proto.beans.QueryResult;
 import org.apache.drill.exec.record.RecordBatchLoader;
 import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.util.JsonStringArrayList;
+import org.apache.drill.exec.util.JsonStringHashMap;
 import org.apache.drill.exec.vector.ValueVector;
 import org.junit.Test;
 
@@ -36,6 +35,55 @@ public class TestJsonReaderWithSparseFiles extends BaseTestQuery {
     void apply(T param);
   }
 
+  static class TypeConverter {
+
+    public Object convert(Object obj) {
+      if (obj == null) {
+        return null;
+      }
+      if (obj instanceof JsonStringArrayList || obj instanceof JsonStringHashMap) {
+        return obj.toString();
+      }
+      return obj;
+    }
+
+  }
+
+  static class Verifier implements Function<RecordBatchLoader> {
+
+    private final int count;
+    private final Object[][] values;
+    private final TypeConverter converter = new TypeConverter();
+
+    protected Verifier(int count, Object[][] values) {
+      this.count = count;
+      this.values = values;
+    }
+
+    @Override
+    public void apply(RecordBatchLoader loader) {
+      assert loader.getRecordCount() == count : "invalid record count returned";
+
+      Object[] row;
+      Object expected;
+      Object actual;
+      for (int r=0;r<values.length;r++) {
+        row = values[r];
+        for (int c=0; c<values[r].length; c++) {
+          expected = row[c];
+          actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r);
+          actual = converter.convert(actual);
+          assert Objects.equals(actual, expected) : String.format("row:%d - col:%d - expected:%s[%s]
- actual:%s[%s]",
+              r, c,
+              expected,
+              expected==null?"null":expected.getClass().getSimpleName(),
+              actual,
+              actual==null?"null":actual.getClass().getSimpleName());
+        }
+      }
+    }
+  }
+
   protected void query(final String query, final Function<RecordBatchLoader> testBody)
throws Exception {
     List<QueryResultBatch> batches = testSqlWithResults(query);
     RecordBatchLoader loader = new RecordBatchLoader(client.getAllocator());
@@ -55,73 +103,37 @@ public class TestJsonReaderWithSparseFiles extends BaseTestQuery {
   @Test
   public void testIfDrillCanReadSparseRecords() throws Exception {
     final String sql = "select * from cp.`vector/complex/fn/sparse.json`";
-    query(sql, new Function<RecordBatchLoader>() {
-      @Override
-      public void apply(RecordBatchLoader loader) {
-        assert loader.getRecordCount() == 4 : "invalid record count returned";
-
-        //XXX: make sure value order matches vector order
-        final Object[][] values = new Object[][] {
-            {null, null},
-            {1L, null},
-            {null, 2L},
-            {3L, 3L}
-        };
-
-        Object[] row;
-        Object expected;
-        Object actual;
-        for (int r=0;r<values.length;r++) {
-          row = values[r];
-          for (int c=0; c<values[r].length; c++) {
-            expected = row[c];
-            actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r);
-            assert Objects.equals(expected, actual) : String.format("row:%d - col:%d - expected:%s[%s]
- actual:%s[%s]",
-                r, c,
-                expected,
-                expected==null?"null":expected.getClass().getSimpleName(),
-                actual,
-                actual==null?"null":actual.getClass().getSimpleName());
-          }
-        }
-      }
-    });
+    //XXX: make sure value order matches vector order
+    final Object[][] values = new Object[][] {
+        {null, null},
+        {1L, null},
+        {null, 2L},
+        {3L, 3L}
+    };
+    query(sql, new Verifier(4, values));
   }
 
   @Test
   public void testIfDrillCanReadSparseNestedRecordsWithoutRaisingException() throws Exception
{
     final String sql = "select * from cp.`vector/complex/fn/nested-with-nulls.json`";
-    query(sql, new Function<RecordBatchLoader>() {
-      @Override
-      public void apply(RecordBatchLoader loader) {
-        assert loader.getRecordCount() == 4 : "invalid record count returned";
-
-        //XXX: make sure value order matches vector order
-        final Object[][] values = new Object[][] {
-            {"[{},{},{},{\"name\":\"doe\"},{}]"},
-            {"[]"},
-            {"[{\"name\":\"john\",\"id\":10}]"},
-            {"[{},{}]"},
-        };
-
-        Object[] row;
-        Object expected;
-        Object actual;
-        for (int r=0;r<values.length;r++) {
-          row = values[r];
-          for (int c = 0; c < values[r].length; c++) {
-            expected = row[c];
-            actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r);
-            assert Objects.equals(actual, expected) : String.format("row:%d - col:%d - expected:%s[%s]
- actual:%s[%s]",
-                r, c,
-                expected,
-                expected == null ? "null" : expected.getClass().getSimpleName(),
-                actual,
-                actual == null ? "null" : actual.getClass().getSimpleName());
-          }
-        }
-      }
-    });
+    //XXX: make sure value order matches vector order
+    final Object[][] values = new Object[][] {
+        {"[{},{},{},{\"name\":\"doe\"},{}]"},
+        {"[]"},
+        {"[{\"name\":\"john\",\"id\":10}]"},
+        {"[{},{}]"},
+    };
+    query(sql, new Verifier(4, values));
+  }
+
+  @Test
+  public void testIfDrillCanQuerySingleRecordWithEmpties() throws Exception {
+    final String sql = "select * from cp.`vector/complex/fn/single-record-with-empties.json`";
+    //XXX: make sure value order matches vector order
+    final Object[][] values = new Object[][] {
+        {"[{},{}]"},
+    };
+    query(sql, new Verifier(1, values));
   }
 
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/a94a028c/exec/java-exec/src/test/resources/vector/complex/fn/single-record-with-empties.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/vector/complex/fn/single-record-with-empties.json
b/exec/java-exec/src/test/resources/vector/complex/fn/single-record-with-empties.json
new file mode 100644
index 0000000..a3c2fbf
--- /dev/null
+++ b/exec/java-exec/src/test/resources/vector/complex/fn/single-record-with-empties.json
@@ -0,0 +1 @@
+{"users":[{}, {"name":null}]}
\ No newline at end of file


Mime
View raw message