Repository: drill Updated Branches: refs/heads/master 6cb626d78 -> 9620e3482
DRILL-5781: Fix unit test failures to use tests config even if default config is available closes #942 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9620e348 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9620e348 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9620e348 Branch: refs/heads/master Commit: 9620e34829a25d767f90b272b9b57ccc9e0e36ea Parents: 6cb626d Author: Volodymyr Vysotskyi <vvo...@gmail.com> Authored: Thu Sep 7 18:01:12 2017 +0000 Committer: Paul Rogers <prog...@maprtech.com> Committed: Sun Sep 24 22:21:20 2017 -0700 ---------------------------------------------------------------------- .../org/apache/drill/hbase/HBaseTestsSuite.java | 5 +- .../src/test/resources/hbase-site.xml | 4 +- .../apache/drill/exec/coord/zk/PathUtils.java | 17 ++++++- .../org/apache/drill/exec/ZookeeperHelper.java | 5 +- .../apache/drill/exec/ZookeeperTestUtil.java | 44 +++++++++++++++++ .../drill/exec/coord/zk/TestEphemeralStore.java | 5 +- .../exec/coord/zk/TestZookeeperClient.java | 3 ++ .../exec/server/TestDrillbitResilience.java | 3 ++ exec/java-exec/src/test/resources/core-site.xml | 52 ++++++++++++++++++++ exec/java-exec/src/test/resources/login.conf | 25 ++++++++++ 10 files changed, 155 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java index 3dd3608..1a176ca 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,6 +21,7 @@ import java.io.IOException; import java.lang.management.ManagementFactory; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.drill.exec.ZookeeperTestUtil; import org.apache.drill.exec.util.GuavaPatcher; import org.apache.drill.hbase.test.Drill2130StorageHBaseHamcrestConfigurationTest; import org.apache.hadoop.conf.Configuration; @@ -95,6 +96,8 @@ public class HBaseTestsSuite { @BeforeClass public static void initCluster() throws Exception { + ZookeeperTestUtil.setJaasTestConfigFile(); + if (initCount.get() == 0) { synchronized (HBaseTestsSuite.class) { if (initCount.get() == 0) { http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/contrib/storage-hbase/src/test/resources/hbase-site.xml ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/resources/hbase-site.xml b/contrib/storage-hbase/src/test/resources/hbase-site.xml index 54425d4..f5c0fcb 100644 --- a/contrib/storage-hbase/src/test/resources/hbase-site.xml +++ b/contrib/storage-hbase/src/test/resources/hbase-site.xml @@ -66,15 +66,13 @@ Default is 10. </description> </property> -<!-- - <property> + <property> <name>hbase.master.info.port</name> <value>-1</value> <description>The port for the hbase master web UI Set to -1 if you do not want the info server to run. </description> </property> ---> <property> <name>hbase.regionserver.info.port</name> <value>-1</value> http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java index f01b989..bc452a9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/coord/zk/PathUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,6 +20,8 @@ package org.apache.drill.exec.coord.zk; import com.google.common.base.Preconditions; import org.apache.parquet.Strings; +import java.net.URL; + /** * A convenience class used to expedite zookeeper paths manipulations. */ @@ -70,4 +72,17 @@ public final class PathUtils { return builder.toString(); } + /** + * Creates and returns path with the protocol at the beginning from specified {@code url}. + * + * @param url the source of path and protocol + * @return string with protocol and path divided by colon + */ + public static String getPathWithProtocol(URL url) { + if (url.getProtocol() != null) { + return url.getProtocol() + ":" + url.getPath(); + } else { + return url.getPath(); + } + } } http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java index 8a07fd2..5c4f83f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperHelper.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -25,7 +25,6 @@ import java.lang.management.ManagementFactory; import java.util.Properties; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.util.MiniZooKeeperCluster; /** @@ -94,6 +93,8 @@ public class ZookeeperHelper { } try { + ZookeeperTestUtil.setZookeeperSaslTestConfigProps(); + zkCluster = new MiniZooKeeperCluster(); zkCluster.setDefaultClientPort(MiniZooKeeperCluster.DEFAULT_PORT); http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java new file mode 100644 index 0000000..20d14de --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ZookeeperTestUtil.java @@ -0,0 +1,44 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to you under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.drill.exec; + +import org.apache.drill.exec.coord.zk.PathUtils; +import org.apache.zookeeper.client.ZooKeeperSaslClient; +import org.apache.zookeeper.server.ZooKeeperSaslServer; + +import static org.apache.zookeeper.Environment.JAAS_CONF_KEY; + +public class ZookeeperTestUtil { + + private static final String LOGIN_CONF_RESOURCE_PATHNAME = "login.conf"; + + /** + * Sets zookeeper server and client SASL test config properties. + */ + public static void setZookeeperSaslTestConfigProps() { + System.setProperty(ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY, "DrillTestServerForUnitTests"); + System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY, "DrillTestClientForUnitTests"); + } + + /** + * Specifies JAAS configuration file. + */ + public static void setJaasTestConfigFile() { + String configPath = PathUtils.getPathWithProtocol(ClassLoader.getSystemResource(LOGIN_CONF_RESOURCE_PATHNAME)); + System.setProperty(JAAS_CONF_KEY, configPath); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java index 021a0b7..3e94589 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestEphemeralStore.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,6 +27,7 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.retry.RetryNTimes; import org.apache.curator.test.TestingServer; +import org.apache.drill.exec.ZookeeperTestUtil; import org.apache.drill.exec.coord.store.TransientStoreConfig; import org.apache.drill.exec.serialization.InstanceSerializer; import org.junit.After; @@ -61,6 +62,8 @@ public class TestEphemeralStore { @Before public void setUp() throws Exception { + ZookeeperTestUtil.setZookeeperSaslTestConfigProps(); + server = new TestingServer(); final RetryPolicy policy = new RetryNTimes(2, 1000); curator = CuratorFrameworkFactory.newClient(server.getConnectString(), policy); http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java index 88f1fcb..a43cee2 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/coord/zk/TestZookeeperClient.java @@ -31,6 +31,7 @@ import org.apache.curator.retry.RetryNTimes; import org.apache.curator.test.TestingServer; import org.apache.drill.common.collections.ImmutableEntry; import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.exec.ZookeeperTestUtil; import org.apache.drill.exec.exception.VersionMismatchException; import org.apache.drill.exec.store.sys.store.DataChangeVersion; import org.apache.zookeeper.CreateMode; @@ -71,6 +72,8 @@ public class TestZookeeperClient { @Before public void setUp() throws Exception { + ZookeeperTestUtil.setZookeeperSaslTestConfigProps(); + server = new TestingServer(); final RetryPolicy policy = new RetryNTimes(1, 1000); curator = CuratorFrameworkFactory.newClient(server.getConnectString(), policy); http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java index 52f5fb9..a823820 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/TestDrillbitResilience.java @@ -42,6 +42,7 @@ import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.util.RepeatTestRule.Repeat; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.ZookeeperHelper; +import org.apache.drill.exec.ZookeeperTestUtil; import org.apache.drill.exec.client.DrillClient; import org.apache.drill.exec.exception.DrillbitStartupException; import org.apache.drill.exec.exception.SchemaChangeException; @@ -185,6 +186,8 @@ public class TestDrillbitResilience extends DrillTest { // turn off the HTTP server to avoid port conflicts between the drill bits System.setProperty(ExecConstants.HTTP_ENABLE, "false"); + ZookeeperTestUtil.setJaasTestConfigFile(); + // turn on error for failure in cancelled fragments zkHelper = new ZookeeperHelper(true, true); zkHelper.startZookeeper(1); http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/resources/core-site.xml ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/core-site.xml b/exec/java-exec/src/test/resources/core-site.xml new file mode 100644 index 0000000..f56d618 --- /dev/null +++ b/exec/java-exec/src/test/resources/core-site.xml @@ -0,0 +1,52 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <property> + <name>fs.hdfs.impl</name> + <value>org.apache.hadoop.hdfs.DistributedFileSystem</value> + </property> + <property> + <name>fs.AbstractFileSystem.hdfs.impl</name> + <value>org.apache.hadoop.hdfs.DistributedFileSystem</value> + <description>The FileSystem for hdfs: uris.</description> + </property> + <property> + <name>dfs.namenode.edits.dir</name> + <value>${dfs.namenode.name.dir}</value> + <description>Determines where on the local filesystem the DFS name node + should store the transaction (edits) file. If this is a comma-delimited list + of directories then the transaction file is replicated in all of the + directories, for redundancy. Default value is same as dfs.namenode.name.dir + </description> + </property> + <property> + <name>fs.defaultFS</name> + <value>file:///</value> + <description>The name of the default file system. A URI whose + scheme and authority determine the FileSystem implementation. The + uri's scheme determines the config property (fs.SCHEME.impl) naming + the FileSystem implementation class. The uri's authority is used to + determine the host, port, etc. for a filesystem.</description> + </property> + <property> + <name>datanucleus.autoCreateTables</name> + <value>true</value> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/9620e348/exec/java-exec/src/test/resources/login.conf ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/login.conf b/exec/java-exec/src/test/resources/login.conf new file mode 100644 index 0000000..0916120 --- /dev/null +++ b/exec/java-exec/src/test/resources/login.conf @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * simple login, just get OS creds + */ +hadoop_simple { + org.apache.hadoop.security.login.GenericOSLoginModule required; + org.apache.hadoop.security.login.HadoopLoginModule required; +}; \ No newline at end of file