[ https://issues.apache.org/jira/browse/DRILL-5126?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15806925#comment-15806925 ]
ASF GitHub Bot commented on DRILL-5126: --------------------------------------- Github user paul-rogers commented on a diff in the pull request: https://github.com/apache/drill/pull/710#discussion_r95052810 --- Diff: exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java --- @@ -0,0 +1,405 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package org.apache.drill.test; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.commons.io.FileUtils; +import org.apache.drill.DrillTestWrapper.TestServices; +import org.apache.drill.QueryTestUtil; +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.ZookeeperHelper; +import org.apache.drill.exec.client.DrillClient; +import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.memory.RootAllocatorFactory; +import org.apache.drill.exec.proto.UserBitShared.QueryType; +import org.apache.drill.exec.rpc.user.QueryDataBatch; +import org.apache.drill.exec.server.Drillbit; +import org.apache.drill.exec.server.RemoteServiceSet; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.StoragePluginRegistryImpl; +import org.apache.drill.exec.store.dfs.FileSystemConfig; +import org.apache.drill.exec.store.dfs.FileSystemPlugin; +import org.apache.drill.exec.store.dfs.WorkspaceConfig; +import org.apache.drill.exec.store.mock.MockStorageEngine; +import org.apache.drill.exec.store.mock.MockStorageEngineConfig; +import org.apache.drill.exec.util.TestUtilities; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import com.google.common.io.Files; +import com.google.common.io.Resources; + +/** + * Test fixture to start a Drillbit with provide options, create a client, + * and execute queries. Can be used in JUnit tests, or in ad-hoc programs. + * Provides a builder to set the necessary embedded Drillbit and client + * options, then creates the requested Drillbit and client. + */ + +public class ClusterFixture implements AutoCloseable { +// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClientFixture.class); + public static final String ENABLE_FULL_CACHE = "drill.exec.test.use-full-cache"; + public static final int MAX_WIDTH_PER_NODE = 2; + + @SuppressWarnings("serial") + public static final Properties TEST_CONFIGURATIONS = new Properties() { + { + // Properties here mimic those in drill-root/pom.xml, Surefire plugin + // configuration. They allow tests to run successfully in Eclipse. + + put(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, false); + put(ExecConstants.HTTP_ENABLE, false); + put(Drillbit.SYSTEM_OPTIONS_NAME, "org.apache.drill.exec.compile.ClassTransformer.scalar_replacement=on"); + put(QueryTestUtil.TEST_QUERY_PRINTING_SILENT, true); + put("drill.catastrophic_to_standard_out", true); + + // See Drillbit.close. The Drillbit normally waits a specified amount + // of time for ZK registration to drop. But, embedded Drillbits normally + // don't use ZK, so no need to wait. + + put(ExecConstants.ZK_REFRESH, 0); + + // This is just a test, no need to be heavy-duty on threads. + // This is the number of server and client RPC threads. The + // production default is DEFAULT_SERVER_RPC_THREADS. + + put(ExecConstants.BIT_SERVER_RPC_THREADS, 2); + + // No need for many scanners except when explicitly testing that + // behavior. Production default is DEFAULT_SCAN_THREADS + + put(ExecConstants.SCAN_THREADPOOL_SIZE, 4); + } + }; + + + public static final String DEFAULT_BIT_NAME = "drillbit"; + + private DrillConfig config; + private Map<String,Drillbit> bits = new HashMap<>( ); + private BufferAllocator allocator; + private boolean ownsZK; + private ZookeeperHelper zkHelper; + private RemoteServiceSet serviceSet; + private String dfsTestTmpSchemaLocation; + List<ClientFixture> clients = new ArrayList<>( ); + + ClusterFixture( FixtureBuilder builder ) throws Exception { + + // Start ZK if requested. + + if (builder.zkHelper != null) { + zkHelper = builder.zkHelper; + ownsZK = false; + } else if (builder.zkCount > 0) { + zkHelper = new ZookeeperHelper(true); + zkHelper.startZookeeper(builder.zkCount); + ownsZK = true; + } + + // Create a config + // Because of the way DrillConfig works, we can set the ZK + // connection string only if a property set is provided. + + if ( builder.configResource != null ) { + config = DrillConfig.create(builder.configResource); + } else if (builder.configProps != null) { + config = DrillConfig.create(configProperties(builder.configProps)); + } else { + config = DrillConfig.create(configProperties(TEST_CONFIGURATIONS)); + } + + // Not quite sure what this is, but some tests seem to use it. + + if (builder.enableFullCache || + (config.hasPath(ENABLE_FULL_CACHE) && config.getBoolean(ENABLE_FULL_CACHE))) { + serviceSet = RemoteServiceSet.getServiceSetWithFullCache(config, allocator); + } else { + serviceSet = RemoteServiceSet.getLocalServiceSet(); + } + + dfsTestTmpSchemaLocation = TestUtilities.createTempDir(); + + Preconditions.checkArgument(builder.bitCount > 0); + int bitCount = builder.bitCount; + for ( int i = 0; i < bitCount; i++ ) { + @SuppressWarnings("resource") + Drillbit bit = new Drillbit(config, serviceSet); + bit.run( ); + + // Create the dfs_test name space + + @SuppressWarnings("resource") + final StoragePluginRegistry pluginRegistry = bit.getContext().getStorage(); + TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, dfsTestTmpSchemaLocation); + TestUtilities.makeDfsTmpSchemaImmutable(pluginRegistry); + + // Create the mock data plugin + // (Disabled until DRILL-5152 is committed.) + +// MockStorageEngineConfig config = MockStorageEngineConfig.INSTANCE; +// @SuppressWarnings("resource") +// MockStorageEngine plugin = new MockStorageEngine(MockStorageEngineConfig.INSTANCE, bit.getContext(), MockStorageEngineConfig.NAME); +// ((StoragePluginRegistryImpl) pluginRegistry).definePlugin(MockStorageEngineConfig.NAME, config, plugin); + + // Bit name and registration. + + String name; + if (builder.bitNames != null && i < builder.bitNames.length) { + name = builder.bitNames[i]; + } else { + if (i == 0) { --- End diff -- We could. But this would mean that in 99% of the tests, the name would be "drillbit1" rather than "drillbit". And, to make it easier to add a second Drillbit to an existing test, if there are two this names them "drillbit" and "drillbit2". Can't recall if I did this to be compatible with older test frameworks. Did clean up the condition and add an explanation. > Provide simplified, unified "cluster fixture" for tests > ------------------------------------------------------- > > Key: DRILL-5126 > URL: https://issues.apache.org/jira/browse/DRILL-5126 > Project: Apache Drill > Issue Type: Improvement > Components: Tools, Build & Test > Affects Versions: 1.9.0 > Reporter: Paul Rogers > Assignee: Paul Rogers > Priority: Minor > > Drill provides a robust selection of test frameworks that have evolved to > satisfy the needs of a variety of test cases. For newbies, however, the > result is a bewildering array of ways to do basically the same thing: set up > an embedded Drill cluster, run queries and check results. > Further, some key test settings are distributed: some are in the pom.xml > file, some in config files stored as resources, some in hard-coded settings > in base test classes. > Also, some test base classes helpfully set up a test cluster, but then > individual tests need a different config, so they immediately tear down the > default cluster and create a new one. > This ticket proposes a new test framework, available for new tests, that > combines the best of the existing test frameworks into a single, easy-to-use > package. > * Builder for the cluster > * Accept config-time options > * Accept run-time session and system options > * Specify number of Drillbits > * Simplified API for the most common options > * AutoCloseable for use in try-with-resources statements > * Integration with existing test builder classes > And so on. -- This message was sent by Atlassian JIRA (v6.3.4#6332)