drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cwestin <...@git.apache.org>
Subject [GitHub] drill pull request: DRILL-3920 Add vector loading tests
Date Tue, 13 Oct 2015 18:51:30 GMT
Github user cwestin commented on a diff in the pull request:

    https://github.com/apache/drill/pull/194#discussion_r41908273
  
    --- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
---
    @@ -223,14 +231,187 @@ public void testNullableVarLen2() {
         }
       }
     
    +  private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[]
buffers) {
    +    // find the total size we'll need
    +    int size = 0;
    +    for(final DrillBuf buffer : buffers) {
    +      size += buffer.readableBytes();
    +    }
    +
    +    // create the new buffer
    +    final DrillBuf newBuf = allocator.buffer(size);
    +    final DrillBuf writeBuf = newBuf;
    +    for(final DrillBuf buffer : buffers) {
    +      final DrillBuf readBuf = (DrillBuf) buffer.slice();
    +      final int nBytes = readBuf.readableBytes();
    +      for(int i = 0; i < nBytes; ++i) {
    +        writeBuf.writeByte(readBuf.readByte());
    +      }
    +    }
    +
    +    return newBuf;
    +  }
    +
    +  @Test
    +  public void testRepeatedIntVector() {
    +    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
    +
    +    // Create a new value vector.
    +    final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
    +
    +    // Populate the vector.
    +    final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
    +    final int nRecords = 7;
    +    final int nElements = values.length;
    +    vector1.allocateNew(nRecords, nRecords * nElements);
    +    final RepeatedIntVector.Mutator mutator = vector1.getMutator();
    +    for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
    +      mutator.startNewValue(recordIndex);
    +      for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
    +        mutator.add(recordIndex, recordIndex * values[elementIndex]);
    +      }
    +    }
    +    mutator.setValueCount(nRecords);
    +
    +    // Verify the contents.
    +    final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
    +    assertEquals(nRecords, accessor1.getValueCount());
    +    for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
    +      for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
    +        final int value = accessor1.get(recordIndex, elementIndex);
    +        assertEquals(recordIndex * values[elementIndex], value);
    +      }
    +    }
    +
    +/* TODO(cwestin)
    +the interface to load has changed
    +    // Serialize, reify, and verify.
    +    final DrillBuf[] buffers1 = vector1.getBuffers(false);
    +    final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
    +    final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
    +    vector2.load(nRecords, nRecords * nElements, buffer1);
    +
    +    final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
    +    for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
    +      for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
    +        final int value = accessor2.get(recordIndex, elementIndex);
    +        assertEquals(accessor1.get(recordIndex,  elementIndex), value);
    +      }
    +    }
    +*/
    +
    +    vector1.close();
    +/* TODO(cwestin)
    +    vector2.close();
    +    buffer1.release();
    +*/
    +  }
    +
    +  @Test
    +  public void testVarCharVectorLoad() {
    +    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
    +
    +    // Create a new value vector for 1024 variable length strings.
    +    final VarCharVector vector1 = new VarCharVector(field, allocator);
    +    final VarCharVector.Mutator mutator = vector1.getMutator();
    +    vector1.allocateNew(1024 * 10, 1024);
    +
    +    // Populate the vector.
    +    final StringBuilder stringBuilder = new StringBuilder();
    +    final int valueCount = 10;
    +    for(int i = 0; i < valueCount; ++i) {
    +      stringBuilder.append('x');
    +      mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
    +    }
    +    mutator.setValueCount(valueCount);
    +    assertEquals(valueCount, vector1.getAccessor().getValueCount());
    +
    +    // Combine the backing buffers so we can load them into a new vector.
    +    final DrillBuf[] buffers1 = vector1.getBuffers(false);
    +    final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
    +    final VarCharVector vector2 = new VarCharVector(field, allocator);
    +    vector2.load(vector1.getMetadata(), buffer1);
    +
    +    // Check the contents of the new vector.
    +    final VarCharVector.Accessor accessor = vector2.getAccessor();
    +    stringBuilder.setLength(0);
    +    for(int i = 0; i < valueCount; ++i) {
    +      stringBuilder.append('x');
    +      final Object object = accessor.getObject(i);
    +      System.out.println(object);
    +      assertEquals(stringBuilder.toString(), object.toString());
    +    }
    +
    +    vector1.close();
    +    vector2.close();
    +    buffer1.release();
    +  }
    +
    +  @Test
    +  public void testNullableVarCharVectorLoad() {
    +    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
    +
    +    // Create a new value vector for 1024 nullable variable length strings.
    +    final NullableVarCharVector vector1 = new NullableVarCharVector(field, allocator);
    +    final NullableVarCharVector.Mutator mutator = vector1.getMutator();
    +    vector1.allocateNew(1024 * 10, 1024);
    +
    +    // Populate the vector.
    +    final StringBuilder stringBuilder = new StringBuilder();
    +    final int valueCount = 10;
    +    for(int i = 0; i < valueCount; ++i) {
    +      stringBuilder.append('x');
    +      mutator.set(i, stringBuilder.toString().getBytes(utf8Charset));
    +    }
    +
    +    // Check the contents.
    +    final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor();
    +    stringBuilder.setLength(0);
    +    for(int i = 0; i < valueCount; ++i) {
    +      stringBuilder.append('x');
    +      final Object object = accessor1.getObject(i);
    +      assertEquals(stringBuilder.toString(), object.toString());
    +    }
    +
    +    mutator.setValueCount(valueCount);
    +    assertEquals(valueCount, vector1.getAccessor().getValueCount());
    +
    +    // Still ok after setting value count?
    +    stringBuilder.setLength(0);
    --- End diff --
    
    It doesn't need to be in both places.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message