drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sohami <...@git.apache.org>
Subject [GitHub] drill pull request #710: DRILL-5126: Provide simplified, unified "cluster fi...
Date Sat, 07 Jan 2017 01:15:49 GMT
Github user sohami commented on a diff in the pull request:

    https://github.com/apache/drill/pull/710#discussion_r95038015
  
    --- Diff: exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java ---
    @@ -0,0 +1,405 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + ******************************************************************************/
    +package org.apache.drill.test;
    +
    +import java.io.File;
    +import java.io.IOException;
    +import java.net.URL;
    +import java.util.ArrayList;
    +import java.util.Collection;
    +import java.util.HashMap;
    +import java.util.List;
    +import java.util.Map;
    +import java.util.Map.Entry;
    +import java.util.Properties;
    +
    +import org.apache.commons.io.FileUtils;
    +import org.apache.drill.DrillTestWrapper.TestServices;
    +import org.apache.drill.QueryTestUtil;
    +import org.apache.drill.common.config.DrillConfig;
    +import org.apache.drill.common.exceptions.ExecutionSetupException;
    +import org.apache.drill.exec.ExecConstants;
    +import org.apache.drill.exec.ZookeeperHelper;
    +import org.apache.drill.exec.client.DrillClient;
    +import org.apache.drill.exec.memory.BufferAllocator;
    +import org.apache.drill.exec.memory.RootAllocatorFactory;
    +import org.apache.drill.exec.proto.UserBitShared.QueryType;
    +import org.apache.drill.exec.rpc.user.QueryDataBatch;
    +import org.apache.drill.exec.server.Drillbit;
    +import org.apache.drill.exec.server.RemoteServiceSet;
    +import org.apache.drill.exec.store.StoragePluginRegistry;
    +import org.apache.drill.exec.store.StoragePluginRegistryImpl;
    +import org.apache.drill.exec.store.dfs.FileSystemConfig;
    +import org.apache.drill.exec.store.dfs.FileSystemPlugin;
    +import org.apache.drill.exec.store.dfs.WorkspaceConfig;
    +import org.apache.drill.exec.store.mock.MockStorageEngine;
    +import org.apache.drill.exec.store.mock.MockStorageEngineConfig;
    +import org.apache.drill.exec.util.TestUtilities;
    +
    +import com.google.common.base.Charsets;
    +import com.google.common.base.Preconditions;
    +import com.google.common.io.Files;
    +import com.google.common.io.Resources;
    +
    +/**
    + * Test fixture to start a Drillbit with provide options, create a client,
    + * and execute queries. Can be used in JUnit tests, or in ad-hoc programs.
    + * Provides a builder to set the necessary embedded Drillbit and client
    + * options, then creates the requested Drillbit and client.
    + */
    +
    +public class ClusterFixture implements AutoCloseable {
    +//  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClientFixture.class);
    +  public static final String ENABLE_FULL_CACHE = "drill.exec.test.use-full-cache";
    +  public static final int MAX_WIDTH_PER_NODE = 2;
    +
    +  @SuppressWarnings("serial")
    +  public static final Properties TEST_CONFIGURATIONS = new Properties() {
    +    {
    +      // Properties here mimic those in drill-root/pom.xml, Surefire plugin
    +      // configuration. They allow tests to run successfully in Eclipse.
    +
    +      put(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, false);
    +      put(ExecConstants.HTTP_ENABLE, false);
    +      put(Drillbit.SYSTEM_OPTIONS_NAME, "org.apache.drill.exec.compile.ClassTransformer.scalar_replacement=on");
    +      put(QueryTestUtil.TEST_QUERY_PRINTING_SILENT, true);
    +      put("drill.catastrophic_to_standard_out", true);
    +
    +      // See Drillbit.close. The Drillbit normally waits a specified amount
    +      // of time for ZK registration to drop. But, embedded Drillbits normally
    +      // don't use ZK, so no need to wait.
    +
    +      put(ExecConstants.ZK_REFRESH, 0);
    +
    +      // This is just a test, no need to be heavy-duty on threads.
    +      // This is the number of server and client RPC threads. The
    +      // production default is DEFAULT_SERVER_RPC_THREADS.
    +
    +      put(ExecConstants.BIT_SERVER_RPC_THREADS, 2);
    +
    +      // No need for many scanners except when explicitly testing that
    +      // behavior. Production default is DEFAULT_SCAN_THREADS
    +
    +      put(ExecConstants.SCAN_THREADPOOL_SIZE, 4);
    +    }
    +  };
    +
    +
    +  public static final String DEFAULT_BIT_NAME = "drillbit";
    +
    +  private DrillConfig config;
    +  private Map<String,Drillbit> bits = new HashMap<>( );
    +  private BufferAllocator allocator;
    +  private boolean ownsZK;
    +  private ZookeeperHelper zkHelper;
    +  private RemoteServiceSet serviceSet;
    +  private String dfsTestTmpSchemaLocation;
    +  List<ClientFixture> clients = new ArrayList<>( );
    +
    +  ClusterFixture( FixtureBuilder  builder ) throws Exception {
    +
    +    // Start ZK if requested.
    +
    +    if (builder.zkHelper != null) {
    +      zkHelper = builder.zkHelper;
    +      ownsZK = false;
    +    } else if (builder.zkCount > 0) {
    +      zkHelper = new ZookeeperHelper(true);
    +      zkHelper.startZookeeper(builder.zkCount);
    +      ownsZK = true;
    +    }
    +
    +    // Create a config
    +    // Because of the way DrillConfig works, we can set the ZK
    +    // connection string only if a property set is provided.
    +
    +    if ( builder.configResource != null ) {
    +      config = DrillConfig.create(builder.configResource);
    +    } else if (builder.configProps != null) {
    +      config = DrillConfig.create(configProperties(builder.configProps));
    +    } else {
    +      config = DrillConfig.create(configProperties(TEST_CONFIGURATIONS));
    +    }
    +
    +    // Not quite sure what this is, but some tests seem to use it.
    +
    +    if (builder.enableFullCache ||
    +        (config.hasPath(ENABLE_FULL_CACHE) && config.getBoolean(ENABLE_FULL_CACHE)))
{
    +      serviceSet = RemoteServiceSet.getServiceSetWithFullCache(config, allocator);
    +    } else {
    +      serviceSet = RemoteServiceSet.getLocalServiceSet();
    +    }
    +
    +    dfsTestTmpSchemaLocation = TestUtilities.createTempDir();
    +
    +    Preconditions.checkArgument(builder.bitCount > 0);
    +    int bitCount = builder.bitCount;
    +    for ( int i = 0;  i < bitCount;  i++ ) {
    +      @SuppressWarnings("resource")
    +      Drillbit bit = new Drillbit(config, serviceSet);
    +      bit.run( );
    +
    +      // Create the dfs_test name space
    +
    +      @SuppressWarnings("resource")
    +      final StoragePluginRegistry pluginRegistry = bit.getContext().getStorage();
    +      TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, dfsTestTmpSchemaLocation);
    +      TestUtilities.makeDfsTmpSchemaImmutable(pluginRegistry);
    +
    +      // Create the mock data plugin
    +      // (Disabled until DRILL-5152 is committed.)
    +
    +//      MockStorageEngineConfig config = MockStorageEngineConfig.INSTANCE;
    +//      @SuppressWarnings("resource")
    +//      MockStorageEngine plugin = new MockStorageEngine(MockStorageEngineConfig.INSTANCE,
bit.getContext(), MockStorageEngineConfig.NAME);
    +//      ((StoragePluginRegistryImpl) pluginRegistry).definePlugin(MockStorageEngineConfig.NAME,
config, plugin);
    +
    +      // Bit name and registration.
    +
    +      String name;
    +      if (builder.bitNames != null && i < builder.bitNames.length) {
    +        name = builder.bitNames[i];
    +      } else {
    +        if (i == 0) {
    +          name = DEFAULT_BIT_NAME;
    +        } else {
    +          name = DEFAULT_BIT_NAME + Integer.toString(i+1);
    +        }
    +      }
    +      bits.put(name, bit);
    +    }
    +
    +    // Some operations need an allocator.
    +
    +    allocator = RootAllocatorFactory.newRoot(config);
    +
    +    // Apply system options
    +    if ( builder.systemOptions != null ) {
    +      for ( FixtureBuilder.RuntimeOption option : builder.systemOptions ) {
    +        clientFixture( ).alterSystem( option.key, option.value );
    +      }
    +    }
    +    // Apply session options.
    +
    +    boolean sawMaxWidth = false;
    +    if ( builder.sessionOptions != null ) {
    +      for ( FixtureBuilder.RuntimeOption option : builder.sessionOptions ) {
    +        clientFixture( ).alterSession( option.key, option.value );
    +        if ( option.key.equals( ExecConstants.MAX_WIDTH_PER_NODE_KEY ) ) {
    +          sawMaxWidth = true;
    +        }
    +      }
    +    }
    +
    +    // Set the default parallelization unless already set by the caller.
    +
    +    if ( ! sawMaxWidth ) {
    +      clientFixture( ).alterSession( ExecConstants.MAX_WIDTH_PER_NODE_KEY, MAX_WIDTH_PER_NODE
);
    +    }
    +  }
    +
    +  private Properties configProperties(Properties configProps) {
    +    Properties effectiveProps = new Properties( );
    +    for (Entry<Object, Object> entry : configProps.entrySet()) {
    +      effectiveProps.put(entry.getKey(), entry.getValue().toString());
    +    }
    +    if (zkHelper != null) {
    +      effectiveProps.put(ExecConstants.ZK_CONNECTION, zkHelper.getConfig().getString(ExecConstants.ZK_CONNECTION));
    +    }
    +    return effectiveProps;
    +  }
    +
    +  public Drillbit drillbit( ) { return bits.get(DEFAULT_BIT_NAME); }
    +  public Drillbit drillbit(String name) { return bits.get(name); }
    +  public Collection<Drillbit> drillbits( ) { return bits.values(); }
    +  public RemoteServiceSet serviceSet( ) { return serviceSet; }
    +  public BufferAllocator allocator( ) { return allocator; }
    +  public DrillConfig config() { return config; }
    +
    +  public ClientFixture.ClientBuilder clientBuilder( ) {
    +    return new ClientFixture.ClientBuilder(this);
    +  }
    +
    +  public ClientFixture clientFixture() {
    +    if ( clients.isEmpty( ) ) {
    +      clientBuilder( ).build( );
    +    }
    +    return clients.get(0);
    +  }
    +
    +  public DrillClient client() {
    +    return clientFixture().client();
    +  }
    +
    +  @Override
    +  public void close() throws Exception {
    +    Exception ex = null;
    +
    +    // Close clients. Clients remove themselves from the client
    +    // list.
    +
    +    while (! clients.isEmpty( )) {
    +      ex = safeClose(clients.get(0), ex);
    +    }
    +
    +    for (Drillbit bit : drillbits()) {
    --- End diff --
    
    Why not use Autocloseables.close instead ?


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message