drill-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (DRILL-3920) Add vector loading tests
Date Mon, 12 Oct 2015 23:23:05 GMT

    [ https://issues.apache.org/jira/browse/DRILL-3920?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14953979#comment-14953979
] 

ASF GitHub Bot commented on DRILL-3920:
---------------------------------------

Github user cwestin commented on a diff in the pull request:

    https://github.com/apache/drill/pull/194#discussion_r41811855
  
    --- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
---
    @@ -223,14 +231,187 @@ public void testNullableVarLen2() {
         }
       }
     
    +  private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[]
buffers) {
    +    // find the total size we'll need
    +    int size = 0;
    +    for(final DrillBuf buffer : buffers) {
    +      size += buffer.readableBytes();
    +    }
    +
    +    // create the new buffer
    +    final DrillBuf newBuf = allocator.buffer(size);
    +    final DrillBuf writeBuf = newBuf;
    +    for(final DrillBuf buffer : buffers) {
    +      final DrillBuf readBuf = (DrillBuf) buffer.slice();
    +      final int nBytes = readBuf.readableBytes();
    +      for(int i = 0; i < nBytes; ++i) {
    +        writeBuf.writeByte(readBuf.readByte());
    +      }
    +    }
    +
    +    return newBuf;
    +  }
    +
    +  @Test
    +  public void testRepeatedIntVector() {
    +    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
    +
    +    // Create a new value vector.
    +    final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
    +
    +    // Populate the vector.
    +    final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
    +    final int nRecords = 7;
    +    final int nElements = values.length;
    +    vector1.allocateNew(nRecords, nRecords * nElements);
    +    final RepeatedIntVector.Mutator mutator = vector1.getMutator();
    +    for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
    +      mutator.startNewValue(recordIndex);
    +      for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
    +        mutator.add(recordIndex, recordIndex * values[elementIndex]);
    +      }
    +    }
    +    mutator.setValueCount(nRecords);
    +
    +    // Verify the contents.
    +    final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
    +    assertEquals(nRecords, accessor1.getValueCount());
    +    for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
    +      for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
    +        final int value = accessor1.get(recordIndex, elementIndex);
    +        assertEquals(recordIndex * values[elementIndex], value);
    +      }
    +    }
    +
    +/* TODO(cwestin)
    +the interface to load has changed
    +    // Serialize, reify, and verify.
    +    final DrillBuf[] buffers1 = vector1.getBuffers(false);
    +    final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
    +    final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
    +    vector2.load(nRecords, nRecords * nElements, buffer1);
    +
    +    final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
    +    for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
    +      for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
    +        final int value = accessor2.get(recordIndex, elementIndex);
    +        assertEquals(accessor1.get(recordIndex,  elementIndex), value);
    +      }
    +    }
    +*/
    +
    +    vector1.close();
    +/* TODO(cwestin)
    +    vector2.close();
    +    buffer1.release();
    +*/
    +  }
    +
    +  @Test
    +  public void testVarCharVectorLoad() {
    +    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
    +
    +    // Create a new value vector for 1024 variable length strings.
    +    final VarCharVector vector1 = new VarCharVector(field, allocator);
    +    final VarCharVector.Mutator mutator = vector1.getMutator();
    +    vector1.allocateNew(1024 * 10, 1024);
    +
    +    // Populate the vector.
    +    final StringBuilder stringBuilder = new StringBuilder();
    +    final int valueCount = 10;
    +    for(int i = 0; i < valueCount; ++i) {
    +      stringBuilder.append('x');
    +      mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
    +    }
    +    mutator.setValueCount(valueCount);
    +    assertEquals(valueCount, vector1.getAccessor().getValueCount());
    +
    +    // Combine the backing buffers so we can load them into a new vector.
    +    final DrillBuf[] buffers1 = vector1.getBuffers(false);
    +    final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
    +    final VarCharVector vector2 = new VarCharVector(field, allocator);
    +    vector2.load(vector1.getMetadata(), buffer1);
    +
    +    // Check the contents of the new vector.
    +    final VarCharVector.Accessor accessor = vector2.getAccessor();
    +    stringBuilder.setLength(0);
    +    for(int i = 0; i < valueCount; ++i) {
    +      stringBuilder.append('x');
    +      final Object object = accessor.getObject(i);
    +      System.out.println(object);
    --- End diff --
    
    Why? This tests something that wasn't working at one point.


> Add vector loading tests
> ------------------------
>
>                 Key: DRILL-3920
>                 URL: https://issues.apache.org/jira/browse/DRILL-3920
>             Project: Apache Drill
>          Issue Type: Bug
>          Components: Execution - Data Types
>    Affects Versions: 1.2.0
>            Reporter: Chris Westin
>            Assignee: Chris Westin
>
> Add some additional tests to TestValueVector to test serialization and reloading of vectors,
as well as the underlying buffer slicing operations that are used for this.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message