Mailing-List: contact issues-help@drill.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@drill.apache.org
Date: Sat, 7 Jan 2017 01:16:01 +0000 (UTC)
From: "ASF GitHub Bot (JIRA)" <jira@apache.org>
To: issues@drill.apache.org
Message-ID: <JIRA.13027533.1481566857000.674302.1483751761311@Atlassian.JIRA>
In-Reply-To: <JIRA.13027533.1481566857000@Atlassian.JIRA>
References: <JIRA.13027533.1481566857000@Atlassian.JIRA> <JIRA.13027533.1481566857634@arcas>
Subject: [jira] [Commented] (DRILL-5126) Provide simplified, unified
 "cluster fixture" for tests
MIME-Version: 1.0
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 7bit
archived-at: Sat, 07 Jan 2017 01:16:03 -0000


    [ https://issues.apache.org/jira/browse/DRILL-5126?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15806397#comment-15806397 ] 

ASF GitHub Bot commented on DRILL-5126:
---------------------------------------

Github user sohami commented on a diff in the pull request:

    https://github.com/apache/drill/pull/710#discussion_r95046976
  
    --- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/xsort/TestSimpleExternalSort.java ---
    @@ -22,220 +22,167 @@
     
     import java.util.List;
     
    -import org.apache.drill.BaseTestQuery;
    -import org.apache.drill.common.config.DrillConfig;
     import org.apache.drill.common.expression.ExpressionPosition;
     import org.apache.drill.common.expression.SchemaPath;
    -import org.apache.drill.common.util.FileUtils;
     import org.apache.drill.common.util.TestTools;
    -import org.apache.drill.exec.client.DrillClient;
    +import org.apache.drill.exec.ExecConstants;
    +import org.apache.drill.exec.exception.SchemaChangeException;
    +import org.apache.drill.exec.memory.BufferAllocator;
     import org.apache.drill.exec.record.RecordBatchLoader;
     import org.apache.drill.exec.rpc.user.QueryDataBatch;
    -import org.apache.drill.exec.server.Drillbit;
    -import org.apache.drill.exec.server.RemoteServiceSet;
     import org.apache.drill.exec.vector.BigIntVector;
    +import org.apache.drill.test.ClientFixture;
    +import org.apache.drill.test.ClusterFixture;
    +import org.apache.drill.test.ClusterTest;
    +import org.apache.drill.test.DrillTest;
    +import org.apache.drill.test.FixtureBuilder;
     import org.junit.Ignore;
     import org.junit.Rule;
     import org.junit.Test;
     import org.junit.rules.TestRule;
     
    -import com.google.common.base.Charsets;
    -import com.google.common.io.Files;
    -
    -@Ignore
    -public class TestSimpleExternalSort extends BaseTestQuery {
    +public class TestSimpleExternalSort extends DrillTest {
       static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleExternalSort.class);
    -  DrillConfig c = DrillConfig.create();
    -
     
       @Rule public final TestRule TIMEOUT = TestTools.getTimeoutRule(80000);
     
       @Test
    -  public void mergeSortWithSv2() throws Exception {
    -    List<QueryDataBatch> results = testPhysicalFromFileWithResults("xsort/one_key_sort_descending_sv2.json");
    -    int count = 0;
    -    for(QueryDataBatch b : results) {
    -      if (b.getHeader().getRowCount() != 0) {
    -        count += b.getHeader().getRowCount();
    -      }
    -    }
    -    assertEquals(500000, count);
    -
    -    long previousBigInt = Long.MAX_VALUE;
    -
    -    int recordCount = 0;
    -    int batchCount = 0;
    -
    -    for (QueryDataBatch b : results) {
    -      if (b.getHeader().getRowCount() == 0) {
    -        break;
    -      }
    -      batchCount++;
    -      RecordBatchLoader loader = new RecordBatchLoader(allocator);
    -      loader.load(b.getHeader().getDef(),b.getData());
    -      BigIntVector c1 = (BigIntVector) loader.getValueAccessorById(BigIntVector.class,
    -              loader.getValueVectorId(new SchemaPath("blue", ExpressionPosition.UNKNOWN)).getFieldIds()).getValueVector();
    -
    -
    -      BigIntVector.Accessor a1 = c1.getAccessor();
    +  public void mergeSortWithSv2Legacy() throws Exception {
    +    mergeSortWithSv2(true);
    +  }
     
    -      for (int i =0; i < c1.getAccessor().getValueCount(); i++) {
    -        recordCount++;
    -        assertTrue(String.format("%d > %d", previousBigInt, a1.get(i)), previousBigInt >= a1.get(i));
    -        previousBigInt = a1.get(i);
    -      }
    -      loader.clear();
    -      b.release();
    +  /**
    +   * Tests the external sort using an in-memory sort. Relies on default memory
    +   * settings to be large enough to do the in-memory sort (there is,
    +   * unfortunately, no way to double-check that no spilling was done.)
    +   * This must be checked manually by setting a breakpoint in the in-memory
    +   * sort routine.
    +   *
    +   * @param testLegacy
    +   * @throws Exception
    +   */
    +
    +  private void mergeSortWithSv2(boolean testLegacy) throws Exception {
    +    try (ClusterFixture cluster = ClusterFixture.standardCluster( );
    +         ClientFixture client = cluster.clientFixture()) {
    +      chooseImpl(client, testLegacy);
    +      List<QueryDataBatch> results = client.queryBuilder().physicalResource("xsort/one_key_sort_descending_sv2.json").results();
    +      assertEquals(500000, client.countResults( results ));
    +      validateResults(client.allocator(), results);
         }
    +  }
     
    -    System.out.println(String.format("Sorted %,d records in %d batches.", recordCount, batchCount));
    +  private void chooseImpl(ClientFixture client, boolean testLegacy) throws Exception {
    --- End diff --
    
    Will the implementation for this be added once the new sort mechanism is checked in ?


> Provide simplified, unified "cluster fixture" for tests
> -------------------------------------------------------
>
>                 Key: DRILL-5126
>                 URL: https://issues.apache.org/jira/browse/DRILL-5126
>             Project: Apache Drill
>          Issue Type: Improvement
>          Components: Tools, Build & Test
>    Affects Versions: 1.9.0
>            Reporter: Paul Rogers
>            Assignee: Paul Rogers
>            Priority: Minor
>
> Drill provides a robust selection of test frameworks that have evolved to satisfy the needs of a variety of test cases. For newbies, however, the result is a bewildering array of ways to do basically the same thing: set up an embedded Drill cluster, run queries and check results.
> Further, some key test settings are distributed: some are in the pom.xml file, some in config files stored as resources, some in hard-coded settings in base test classes.
> Also, some test base classes helpfully set up a test cluster, but then individual tests need a different config, so they immediately tear down the default cluster and create a new one.
> This ticket proposes a new test framework, available for new tests, that combines the best of the existing test frameworks into a single, easy-to-use package.
> * Builder for the cluster
> * Accept config-time options
> * Accept run-time session and system options
> * Specify number of Drillbits
> * Simplified API for the most common options
> * AutoCloseable for use in try-with-resources statements
> * Integration with existing test builder classes
> And so on.


--
This message was sent by Atlassian JIRA
(v6.3.4#6332)