drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From paul-rogers <...@git.apache.org>
Subject [GitHub] drill pull request #753: DRILL-5260: Extend "Cluster Fixture" test framework
Date Thu, 23 Feb 2017 23:53:26 GMT
Github user paul-rogers commented on a diff in the pull request:

    https://github.com/apache/drill/pull/753#discussion_r102843167
  
    --- Diff: exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java ---
    @@ -99,33 +111,83 @@
           // behavior. Production default is DEFAULT_SCAN_THREADS
     
           put(ExecConstants.SCAN_THREADPOOL_SIZE, 4);
    +
    +      // Define a useful root location for the ZK persistent
    +      // storage. Profiles will go here when running in distributed
    +      // mode.
    +
    +      put(ZookeeperPersistentStoreProvider.DRILL_EXEC_SYS_STORE_PROVIDER_ZK_BLOBROOT,
"/tmp/drill/log");
         }
       };
     
       public static final String DEFAULT_BIT_NAME = "drillbit";
     
       private DrillConfig config;
    -  private Map<String,Drillbit> bits = new HashMap<>();
    +  private Map<String, Drillbit> bits = new HashMap<>();
       private Drillbit defaultDrillbit;
       private BufferAllocator allocator;
       private boolean ownsZK;
       private ZookeeperHelper zkHelper;
       private RemoteServiceSet serviceSet;
    -  private String dfsTestTmpSchemaLocation;
    +  private File dfsTestTempDir;
       protected List<ClientFixture> clients = new ArrayList<>();
    +  private boolean usesZk;
    +  private boolean preserveLocalFiles;
    +  private boolean isLocal;
    +
    +  /**
    +   * Temporary directories created for this test cluster.
    +   * Each is removed when closing the cluster.
    +   */
    +
    +  private List<File> tempDirs = new ArrayList<>();
    +
    +  ClusterFixture(FixtureBuilder builder) {
    +
    +    String zkConnect = configureZk(builder);
    +    try {
    +      createConfig(builder, zkConnect);
    +      startDrillbits(builder);
    +      applyOptions(builder);
    +    } catch (Exception e) {
    +      // Translate exceptions to unchecked to avoid cluttering
    +      // tests. Failures will simply fail the test itself.
    +
    +      throw new IllegalStateException( "Cluster fixture setup failed", e );
    +    }
    +
    +    // Some operations need an allocator.
     
    -  protected ClusterFixture(FixtureBuilder  builder) throws Exception {
    +    allocator = RootAllocatorFactory.newRoot(config);
    +  }
    +
    +  private String configureZk(FixtureBuilder builder) {
     
         // Start ZK if requested.
     
    +    String zkConnect = null;
         if (builder.zkHelper != null) {
    +      // Case where the test itself started ZK and we're only using it.
    +
           zkHelper = builder.zkHelper;
           ownsZK = false;
    -    } else if (builder.zkCount > 0) {
    -      zkHelper = new ZookeeperHelper(true);
    -      zkHelper.startZookeeper(builder.zkCount);
    +    } else if (builder.localZkCount > 0) {
    +      // Case where we need a local ZK just for this test cluster.
    +
    +      zkHelper = new ZookeeperHelper("dummy");
    +      zkHelper.startZookeeper(builder.localZkCount);
           ownsZK = true;
         }
    +    if (zkHelper != null) {
    +      zkConnect = zkHelper.getConnectionString();
    +      // Forced to disable this, because currently we leak memory which is a known issue
for query cancellations.
    +      // Setting this causes unittests to fail.
    +      builder.configProperty(ExecConstants.RETURN_ERROR_FOR_FAILURE_IN_CANCELLED_FRAGMENTS,
true);
    +    }
    +    return zkConnect;
    --- End diff --
    
    Good catch. Fixed.
    
    Lots of this code was adapted from elsewhere and I didn't always follow through the exact
flow. Thanks for doing so.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message