drill-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (DRILL-5126) Provide simplified, unified "cluster fixture" for tests
Date Sat, 07 Jan 2017 06:27:59 GMT

    [ https://issues.apache.org/jira/browse/DRILL-5126?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15806927#comment-15806927
] 

ASF GitHub Bot commented on DRILL-5126:
---------------------------------------

Github user paul-rogers commented on a diff in the pull request:

    https://github.com/apache/drill/pull/710#discussion_r95052757
  
    --- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java
---
    @@ -22,220 +22,167 @@
     
     import java.util.List;
     
    -import org.apache.drill.BaseTestQuery;
    -import org.apache.drill.common.config.DrillConfig;
     import org.apache.drill.common.expression.ExpressionPosition;
     import org.apache.drill.common.expression.SchemaPath;
    -import org.apache.drill.common.util.FileUtils;
     import org.apache.drill.common.util.TestTools;
    -import org.apache.drill.exec.client.DrillClient;
    +import org.apache.drill.exec.ExecConstants;
    +import org.apache.drill.exec.exception.SchemaChangeException;
    +import org.apache.drill.exec.memory.BufferAllocator;
     import org.apache.drill.exec.record.RecordBatchLoader;
     import org.apache.drill.exec.rpc.user.QueryDataBatch;
    -import org.apache.drill.exec.server.Drillbit;
    -import org.apache.drill.exec.server.RemoteServiceSet;
     import org.apache.drill.exec.vector.BigIntVector;
    +import org.apache.drill.test.ClientFixture;
    +import org.apache.drill.test.ClusterFixture;
    +import org.apache.drill.test.ClusterTest;
    +import org.apache.drill.test.DrillTest;
    +import org.apache.drill.test.FixtureBuilder;
     import org.junit.Ignore;
     import org.junit.Rule;
     import org.junit.Test;
     import org.junit.rules.TestRule;
     
    -import com.google.common.base.Charsets;
    -import com.google.common.io.Files;
    -
    -@Ignore
    -public class TestSimpleExternalSort extends BaseTestQuery {
    +public class TestSimpleExternalSort extends DrillTest {
       static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleExternalSort.class);
    -  DrillConfig c = DrillConfig.create();
    -
     
       @Rule public final TestRule TIMEOUT = TestTools.getTimeoutRule(80000);
     
       @Test
    -  public void mergeSortWithSv2() throws Exception {
    -    List<QueryDataBatch> results = testPhysicalFromFileWithResults("xsort/one_key_sort_descending_sv2.json");
    -    int count = 0;
    -    for(QueryDataBatch b : results) {
    -      if (b.getHeader().getRowCount() != 0) {
    -        count += b.getHeader().getRowCount();
    -      }
    -    }
    -    assertEquals(500000, count);
    -
    -    long previousBigInt = Long.MAX_VALUE;
    -
    -    int recordCount = 0;
    -    int batchCount = 0;
    -
    -    for (QueryDataBatch b : results) {
    -      if (b.getHeader().getRowCount() == 0) {
    -        break;
    -      }
    -      batchCount++;
    -      RecordBatchLoader loader = new RecordBatchLoader(allocator);
    -      loader.load(b.getHeader().getDef(),b.getData());
    -      BigIntVector c1 = (BigIntVector) loader.getValueAccessorById(BigIntVector.class,
    -              loader.getValueVectorId(new SchemaPath("blue", ExpressionPosition.UNKNOWN)).getFieldIds()).getValueVector();
    -
    -
    -      BigIntVector.Accessor a1 = c1.getAccessor();
    +  public void mergeSortWithSv2Legacy() throws Exception {
    +    mergeSortWithSv2(true);
    +  }
     
    -      for (int i =0; i < c1.getAccessor().getValueCount(); i++) {
    -        recordCount++;
    -        assertTrue(String.format("%d > %d", previousBigInt, a1.get(i)), previousBigInt
>= a1.get(i));
    -        previousBigInt = a1.get(i);
    -      }
    -      loader.clear();
    -      b.release();
    +  /**
    +   * Tests the external sort using an in-memory sort. Relies on default memory
    +   * settings to be large enough to do the in-memory sort (there is,
    +   * unfortunately, no way to double-check that no spilling was done.)
    +   * This must be checked manually by setting a breakpoint in the in-memory
    +   * sort routine.
    +   *
    +   * @param testLegacy
    +   * @throws Exception
    +   */
    +
    +  private void mergeSortWithSv2(boolean testLegacy) throws Exception {
    +    try (ClusterFixture cluster = ClusterFixture.standardCluster( );
    +         ClientFixture client = cluster.clientFixture()) {
    +      chooseImpl(client, testLegacy);
    +      List<QueryDataBatch> results = client.queryBuilder().physicalResource("xsort/one_key_sort_descending_sv2.json").results();
    +      assertEquals(500000, client.countResults( results ));
    +      validateResults(client.allocator(), results);
         }
    +  }
     
    -    System.out.println(String.format("Sorted %,d records in %d batches.", recordCount,
batchCount));
    +  private void chooseImpl(ClientFixture client, boolean testLegacy) throws Exception
{
    --- End diff --
    
    Yes. This is a placeholder. Once the managed sort and this framework are both committed,
then this test will be replaced with one that tests both the original and managed version.
This method switches between them.


> Provide simplified, unified "cluster fixture" for tests
> -------------------------------------------------------
>
>                 Key: DRILL-5126
>                 URL: https://issues.apache.org/jira/browse/DRILL-5126
>             Project: Apache Drill
>          Issue Type: Improvement
>          Components: Tools, Build & Test
>    Affects Versions: 1.9.0
>            Reporter: Paul Rogers
>            Assignee: Paul Rogers
>            Priority: Minor
>
> Drill provides a robust selection of test frameworks that have evolved to satisfy the
needs of a variety of test cases. For newbies, however, the result is a bewildering array
of ways to do basically the same thing: set up an embedded Drill cluster, run queries and
check results.
> Further, some key test settings are distributed: some are in the pom.xml file, some in
config files stored as resources, some in hard-coded settings in base test classes.
> Also, some test base classes helpfully set up a test cluster, but then individual tests
need a different config, so they immediately tear down the default cluster and create a new
one.
> This ticket proposes a new test framework, available for new tests, that combines the
best of the existing test frameworks into a single, easy-to-use package.
> * Builder for the cluster
> * Accept config-time options
> * Accept run-time session and system options
> * Specify number of Drillbits
> * Simplified API for the most common options
> * AutoCloseable for use in try-with-resources statements
> * Integration with existing test builder classes
> And so on.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message