Repository: drill Updated Branches: refs/heads/master cf2478f7a -> 36159e228
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/TestScripts.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/TestScripts.java b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/TestScripts.java new file mode 100644 index 0000000..e8f30a6 --- /dev/null +++ b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/TestScripts.java @@ -0,0 +1,1288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.scripts; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; + +import org.apache.drill.yarn.scripts.ScriptUtils.DrillbitRun; +import org.apache.drill.yarn.scripts.ScriptUtils.RunResult; +import org.apache.drill.yarn.scripts.ScriptUtils.ScriptRunner; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +/** + * Unit tests to test the many ways that the Drill shell scripts can run. + * Since it would be difficult to test options using the actual Drillbit, the + * scripts make use of a special test fixture in runbit: the ability to pass + * a "wrapper" script to run in place of the Drillit. That script probes stderr, + * stdout and log files, and writes its arguments (which is the Drillbit launch + * command) to a file. As a result, we can capture this output and analyze it + * to ensure we are passing the right arguments to the Drillbit, and that output + * is going to the right destinations. + */ + +// Turned of by default: works only in a developer setup +@Ignore +public class TestScripts { + static ScriptUtils context; + + @BeforeClass + public static void initialSetup() throws IOException { + context = ScriptUtils.instance(); + context.initialSetup(); + } + + /** + * Test the simplest case: use the $DRILL_HOME/conf directory and default log + * location. Non-existent drill-env.sh and drill-config.sh files. Everything + * is at its Drill-provided defaults. Then, try overriding each user-settable + * environment variable in the environment (which simulates what YARN might + * do.) + */ + + @Test + public void testStockCombined() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + + // No drill-env.sh, no distrib-env.sh + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .run(); + assertEquals(0, result.returnCode); + result.validateJava(); + result.validateStockArgs(); + result.validateClassPath(ScriptUtils.stdCp); + result.validateStdOut(); + result.validateStdErr(); + result.validateDrillLog(); + } + + // As above, but pass an argument. + + { + String propArg = "-Dproperty=value"; + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withArg(propArg).run(); + assertEquals(0, result.returnCode); + result.validateStdOut(); + result.validateArg(propArg); + } + + // Custom Java opts to achieve the same result + + { + String propArg = "-Dproperty=value"; + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILL_JAVA_OPTS", propArg).run(); + assertEquals(0, result.returnCode); + result.validateStockArgs(); // Should not lose standard JVM args + result.validateStdOut(); + result.validateArg(propArg); + } + + // Custom Drillbit Java Opts to achieve the same result + + { + String propArg = "-Dproperty2=value2"; + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILLBIT_JAVA_OPTS", propArg).run(); + assertEquals(0, result.returnCode); + result.validateStockArgs(); // Should not lose standard JVM args + result.validateStdOut(); + result.validateArg(propArg); + } + + // Both sets of options + + { + String propArg = "-Dproperty=value"; + String propArg2 = "-Dproperty2=value2"; + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILL_JAVA_OPTS", propArg) + .addEnv("DRILLBIT_JAVA_OPTS", propArg2).run(); + assertEquals(0, result.returnCode); + result.validateArgs(new String[] { propArg, propArg2 }); + } + + // Custom heap memory + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILL_HEAP", "5G").run(); + result.validateArgs(new String[] { "-Xms5G", "-Xmx5G" }); + } + + // Custom direct memory + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILL_MAX_DIRECT_MEMORY", "7G").run(); + result.validateArg("-XX:MaxDirectMemorySize=7G"); + } + + // Enable GC logging + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("SERVER_LOG_GC", "1").run(); + String logTail = context.testLogDir.getName() + "/drillbit.gc"; + result.validateArgRegex("-Xloggc:.*/" + logTail); + } + + // Max Perm Size + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILLBIT_MAX_PERM", "600M").run(); + result.validateArg("-XX:MaxPermSize=600M"); + } + + // Code cache size + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILLBIT_CODE_CACHE_SIZE", "2G").run(); + result.validateArg("-XX:ReservedCodeCacheSize=2G"); + } + } + + /** + * Use the "stock" setup, but add each custom bit of the class path to ensure + * it is passed to the Drillbit. + * + * @throws IOException + */ + + @Test + public void testClassPath() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + + File extrasDir = context.createDir(new File(context.testDir, "extras")); + File hadoopJar = context.makeDummyJar(extrasDir, "hadoop"); + File hbaseJar = context.makeDummyJar(extrasDir, "hbase"); + File prefixJar = context.makeDummyJar(extrasDir, "prefix"); + File cpJar = context.makeDummyJar(extrasDir, "cp"); + File extnJar = context.makeDummyJar(extrasDir, "extn"); + File toolsJar = context.makeDummyJar(extrasDir, "tools"); + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILL_CLASSPATH_PREFIX", prefixJar.getAbsolutePath()).run(); + result.validateClassPath(prefixJar.getAbsolutePath()); + } + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILL_TOOL_CP", toolsJar.getAbsolutePath()).run(); + result.validateClassPath(toolsJar.getAbsolutePath()); + } + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("HADOOP_CLASSPATH", hadoopJar.getAbsolutePath()).run(); + result.validateClassPath(hadoopJar.getAbsolutePath()); + } + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("HBASE_CLASSPATH", hbaseJar.getAbsolutePath()).run(); + result.validateClassPath(hbaseJar.getAbsolutePath()); + } + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("EXTN_CLASSPATH", extnJar.getAbsolutePath()).run(); + result.validateClassPath(extnJar.getAbsolutePath()); + } + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILL_CLASSPATH", cpJar.getAbsolutePath()).run(); + result.validateClassPath(cpJar.getAbsolutePath()); + } + + // Site jars not on path if not created + + File siteJars = new File(siteDir, "jars"); + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN).run(); + assertFalse(result.classPathContains(siteJars.getAbsolutePath())); + } + + // Site/jars on path if exists + + context.createDir(siteJars); + context.makeDummyJar(siteJars, "site"); + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN).run(); + result.validateClassPath(siteJars.getAbsolutePath() + "/*"); + } + } + + /** + * Create a custom log folder location. + */ + + @Test + public void testLogDir() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + File logsDir = context.createDir(new File(context.testDir, "logs")); + context.removeDir(new File(context.testDrillHome, "log")); + + { + String logPath = logsDir.getAbsolutePath(); + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILL_LOG_DIR", logPath).withLogDir(logsDir).run(); + assertEquals(0, result.returnCode); + result.validateArgs( + new String[] { "-Dlog.path=" + logPath + "/drillbit.log", + "-Dlog.query.path=" + logPath + "/drillbit_queries.json", }); + result.validateStdOut(); + result.validateStdErr(); + result.validateDrillLog(); + } + + } + + /** + * Create a custom Java lib path. This uses the new DRILL_JAVA_LIB_PATH + * variable. + */ + + @Test + public void testLibPath() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + File logsDir = context.createDir(new File(context.testDir, "logs")); + context.removeDir(new File(context.testDrillHome, "log")); + + { + String logPath = logsDir.getAbsolutePath(); + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILL_LOG_DIR", logPath).withLogDir(logsDir).run(); + assertEquals(0, result.returnCode); + result.validateArgs( + new String[] { "-Dlog.path=" + logPath + "/drillbit.log", + "-Dlog.query.path=" + logPath + "/drillbit_queries.json", }); + result.validateStdOut(); + result.validateStdErr(); + result.validateDrillLog(); + } + + } + + /** + * Try setting custom environment variable values in drill-env.sh in the + * $DRILL_HOME/conf location. + */ + + @Test + public void testDrillEnv() throws IOException { + doEnvFileTest("drill-env.sh"); + } + + /** + * Repeat the above test using distrib-env.sh in the $DRILL_HOME/conf + * location. + */ + + @Test + public void testDistribEnv() throws IOException { + doEnvFileTest("distrib-env.sh"); + } + + /** + * Implementation of the drill-env.sh and distrib-env.sh tests. + */ + + private void doEnvFileTest(String fileName) throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + + /** + * Set all properties in the env file. + */ + + Map<String, String> drillEnv = new HashMap<>(); + String propArg = "-Dproperty=value"; + drillEnv.put("DRILL_JAVA_OPTS", propArg); + drillEnv.put("DRILL_HEAP", "5G"); + drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G"); + drillEnv.put("SERVER_LOG_GC", "1"); + drillEnv.put("DRILLBIT_MAX_PERM", "600M"); + drillEnv.put("DRILLBIT_CODE_CACHE_SIZE", "2G"); + context.createEnvFile(new File(siteDir, fileName), drillEnv); + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN).run(); + assertEquals(0, result.returnCode); + + String expectedArgs[] = { + propArg, + "-Xms5G", "-Xmx5G", + "-XX:MaxDirectMemorySize=7G", + "-XX:ReservedCodeCacheSize=2G", + "-XX:MaxPermSize=600M" + }; + + result.validateArgs(expectedArgs); + String logTail = context.testLogDir.getName() + "/drillbit.gc"; + result.validateArgRegex("-Xloggc:.*/" + logTail); + } + + // Change some drill-env.sh options in the environment. + // The generated drill-env.sh should allow overrides. + // (The generated form is the form that customers should use.) + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("SERVER_LOG_GC", "0") + .addEnv("DRILL_MAX_DIRECT_MEMORY", "9G") + .run(); + assertEquals(0, result.returnCode); + result.validateArg("-XX:MaxDirectMemorySize=9G"); + result.validateArg("-XX:MaxPermSize=600M"); + String logTail = context.testDrillHome.getName() + "/log/drillbit.gc"; + assertFalse(result.containsArgRegex("-Xloggc:.*/" + logTail)); + } + } + + /** + * Test that drill-env.sh overrides distrib-env.sh, and that the environment + * overrides both. Assumes the basics were tested above. + * + * @throws IOException + */ + + @Test + public void testDrillAndDistribEnv() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + + Map<String, String> distribEnv = new HashMap<>(); + distribEnv.put("DRILL_HEAP", "5G"); + distribEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G"); + distribEnv.put("DRILLBIT_MAX_PERM", "600M"); + context.createEnvFile(new File(siteDir, "distrib-env.sh"), distribEnv); + + Map<String, String> drillEnv = new HashMap<>(); + drillEnv.put("DRILL_HEAP", "6G"); + drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "9G"); + context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv); + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN).run(); + assertEquals(0, result.returnCode); + String expectedArgs[] = { + "-Xms6G", "-Xmx6G", + "-XX:MaxDirectMemorySize=9G", + "-XX:MaxPermSize=600M", + "-XX:ReservedCodeCacheSize=1G" // Default + }; + + result.validateArgs(expectedArgs); + } + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILL_MAX_DIRECT_MEMORY", "5G").run(); + assertEquals(0, result.returnCode); + String expectedArgs[] = { + "-Xms6G", "-Xmx6G", + "-XX:MaxDirectMemorySize=5G", + "-XX:MaxPermSize=600M", + "-XX:ReservedCodeCacheSize=1G" // Default + }; + + result.validateArgs(expectedArgs); + } + } + + @Test + public void testBadSiteDir() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.removeDir(siteDir); + + // Directory does not exist. + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withSite(siteDir).run(); + assertEquals(1, result.returnCode); + assertTrue(result.stderr.contains("Config dir does not exist")); + } + + // Not a directory + + context.writeFile(siteDir, "dummy"); + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withSite(siteDir).run(); + assertEquals(1, result.returnCode); + assertTrue(result.stderr.contains("Config dir does not exist")); + } + + // Directory exists, but drill-override.conf does not + + siteDir.delete(); + context.createDir(siteDir); + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withSite(siteDir).run(); + assertEquals(1, result.returnCode); + assertTrue(result.stderr.contains("Drill config file missing")); + } + } + + /** + * Move configuration to a site folder out of $DRILL_HOME/conf. The site + * folder can contain code (which is why we call it "site" and not "config".) + * The site directory can be passed to the Drillbit in several ways. + * + * @throws IOException + */ + + @Test + public void testSiteDir() throws IOException { + context.createMockDistrib(); + File confDir = new File(context.testDrillHome, "conf"); + context.createDir(confDir); + File siteDir = new File(context.testDir, "site"); + context.createMockConf(siteDir); + + // Dummy drill-env.sh to simulate the shipped "example" file. + + context.writeFile(new File(confDir, "drill-env.sh"), + "#!/usr/bin/env bash\n" + "# Example file"); + File siteJars = new File(siteDir, "jars"); + + Map<String, String> distribEnv = new HashMap<>(); + distribEnv.put("DRILL_HEAP", "5G"); + distribEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G"); + distribEnv.put("DRILLBIT_MAX_PERM", "600M"); + context.createEnvFile(new File(confDir, "distrib-env.sh"), distribEnv); + + Map<String, String> drillEnv = new HashMap<>(); + drillEnv.put("DRILL_HEAP", "6G"); + drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "9G"); + context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv); + + String expectedArgs[] = { + "-Xms6G", "-Xmx6G", + "-XX:MaxDirectMemorySize=9G", + "-XX:MaxPermSize=600M", + "-XX:ReservedCodeCacheSize=1G" // Default + }; + + // Site set using argument + + { + // Use --config explicitly + + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withArg("--config") + .withArg(siteDir.getAbsolutePath()) + .run(); + assertEquals(0, result.returnCode); + result.validateArgs(expectedArgs); + result.validateClassPath(siteDir.getAbsolutePath()); + } + + { + RunResult result = new DrillbitRun() + .withArg("--config") + .withArg(siteDir.getAbsolutePath()) + .withArg(DrillbitRun.DRILLBIT_RUN) + .run(); + assertEquals(0, result.returnCode); + result.validateArgs(expectedArgs); + } + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withSite(siteDir) + .run(); + assertEquals(0, result.returnCode); + result.validateArgs(expectedArgs); + } + + // Site argument and argument to Drillbit + + { + String propArg = "-Dproperty=value"; + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withSite(siteDir) + .withArg(propArg) + .run( ); + assertEquals(0, result.returnCode); + result.validateArgs(expectedArgs); + result.validateArg(propArg); + } + + // Set as an environment variable + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .addEnv("DRILL_CONF_DIR",siteDir.getAbsolutePath()) + .run(); + assertEquals(0, result.returnCode); + result.validateArgs(expectedArgs); + } + + // Site jars not on path if not created + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withSite(siteDir) + .run(); + assertFalse(result.classPathContains(siteJars.getAbsolutePath())); + } + + // Site/jars on path if exists + + context.createDir(siteJars); + context.makeDummyJar(siteJars, "site"); + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withSite(siteDir) + .run(); + assertTrue(result.classPathContains(siteJars.getAbsolutePath() + "/*")); + } + } + + /** + * Test the Java library path. Three sources: + * <ol> + * <li>DRILL_JAVA_LIB_PATH Set in drill-env.sh</li> + * <li>DOY_JAVA_LIB_PATH passed in from an env. var.</li> + * <li>$DRILL_SITE/lib, if it exists.</li> + * </ol> + * + * @throws IOException + */ + + @Test + public void testJavaLibDir() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + + // Stock run: no lib dir. + + String prefix = "-Djava.library.path="; + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .run(); + assertFalse(result.containsArgRegex(prefix + ".*")); + assertNull(result.libPath); + } + + // Old-style argument in DRILL_JAVA_OPTS + + { + Map<String, String> env = new HashMap<>(); + env.put("DRILL_JAVA_OPTS", prefix + "/foo/bar:/foo/mumble"); + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withEnvironment(env) + .run(); + assertTrue(result.containsArgRegex(prefix + ".*")); + assertNotNull(result.libPath); + assertEquals(2, result.libPath.length); + assertEquals("/foo/bar", result.libPath[0]); + assertEquals("/foo/mumble", result.libPath[1]); + } + + // New-style argument in DRILL_JAVA_LIB_PATH + + { + Map<String, String> env = new HashMap<>(); + env.put("DRILL_JAVA_LIB_PATH", "/foo/bar:/foo/mumble"); + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withEnvironment(env) + .run(); + assertTrue(result.containsArgRegex(prefix + ".*")); + assertNotNull(result.libPath); + assertEquals(2, result.libPath.length); + assertEquals("/foo/bar", result.libPath[0]); + assertEquals("/foo/mumble", result.libPath[1]); + } + + // YARN argument in DOY_JAVA_LIB_PATH + + { + Map<String, String> env = new HashMap<>(); + env.put("DOY_JAVA_LIB_PATH", "/foo/bar:/foo/mumble"); + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withEnvironment(env) + .run(); + assertTrue(result.containsArgRegex(prefix + ".*")); + assertNotNull(result.libPath); + assertEquals(2, result.libPath.length); + assertEquals("/foo/bar", result.libPath[0]); + assertEquals("/foo/mumble", result.libPath[1]); + } + + // Both DRILL_JAVA_LIB_PATH and DOY_JAVA_LIB_PATH + + { + Map<String, String> env = new HashMap<>(); + env.put("DRILL_JAVA_LIB_PATH", "/foo/bar:/foo/mumble"); + env.put("DOY_JAVA_LIB_PATH", "/doy/bar:/doy/mumble"); + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withEnvironment(env) + .run(); + assertTrue(result.containsArgRegex(prefix + ".*")); + assertNotNull(result.libPath); + assertEquals(4, result.libPath.length); + assertEquals("/doy/bar", result.libPath[0]); + assertEquals("/doy/mumble", result.libPath[1]); + assertEquals("/foo/bar", result.libPath[2]); + assertEquals("/foo/mumble", result.libPath[3]); + } + + // Site directory with a lib folder + + siteDir = new File(context.testDir, "site"); + context.createMockConf(siteDir); + File libDir = new File(siteDir, "lib"); + context.createDir(libDir); + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withSite(siteDir) + .run(); + assertTrue(result.containsArgRegex(prefix + ".*")); + assertNotNull(result.libPath); + assertEquals(1, result.libPath.length); + assertEquals(libDir.getAbsolutePath(), result.libPath[0]); + } + + // The whole enchilada: all three settings. + + { + Map<String, String> env = new HashMap<>(); + env.put("DRILL_JAVA_LIB_PATH", "/foo/bar:/foo/mumble"); + env.put("DOY_JAVA_LIB_PATH", "/doy/bar:/doy/mumble"); + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RUN) + .withSite(siteDir) + .withEnvironment(env) + .run(); + assertTrue(result.containsArgRegex(prefix + ".*")); + assertNotNull(result.libPath); + assertEquals(5, result.libPath.length); + assertEquals(libDir.getAbsolutePath(), result.libPath[0]); + assertEquals("/doy/bar", result.libPath[1]); + assertEquals("/doy/mumble", result.libPath[2]); + assertEquals("/foo/bar", result.libPath[3]); + assertEquals("/foo/mumble", result.libPath[4]); + } + } + + /** + * Test running a (simulated) Drillbit as a daemon with start, status, stop. + * + * @throws IOException + */ + + @Test + public void testStockDaemon() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + + // No drill-env.sh, no distrib-env.sh + + File pidFile; + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_START).start(); + assertEquals(0, result.returnCode); + result.validateJava(); + result.validateStockArgs(); + result.validateClassPath(ScriptUtils.stdCp); + assertTrue(result.stdout.contains("Starting drillbit, logging")); + assertTrue(result.log.contains("Starting drillbit on")); + assertTrue(result.log.contains("Drill Log Message")); + assertTrue(result.out.contains("Drill Stdout Message")); + assertTrue(result.out.contains("Stderr Message")); + pidFile = result.pidFile; + } + + // Save the pid file for reuse. + + assertTrue(pidFile.exists()); + File saveDir = new File(context.testDir, "save"); + context.createDir(saveDir); + File savedPidFile = new File(saveDir, pidFile.getName()); + context.copyFile(pidFile, savedPidFile); + + // Status should be running + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STATUS).run( ); + assertEquals(0, result.returnCode); + assertTrue(result.stdout.contains("drillbit is running")); + } + + // Start should refuse to start a second Drillbit. + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_START).start(); + assertEquals(1, result.returnCode); + assertTrue( + result.stdout.contains("drillbit is already running as process")); + } + + // Normal start, allow normal shutdown + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STOP).run(); + assertEquals(0, result.returnCode); + assertTrue(result.log.contains("Terminating drillbit pid")); + assertTrue(result.stdout.contains("Stopping drillbit")); + } + + // Status should report no drillbit (no pid file) + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STATUS).run(); + assertEquals(1, result.returnCode); + assertTrue(result.stdout.contains("drillbit is not running")); + } + + // Stop should report no pid file + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STOP).run(); + assertEquals(1, result.returnCode); + assertTrue( + result.stdout.contains("No drillbit to stop because no pid file")); + } + + // Get nasty. Put the pid file back. But, there is no process with that pid. + + context.copyFile(savedPidFile, pidFile); + + // Status should now complain. + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STATUS).run(); + assertEquals(1, result.returnCode); + assertTrue(result.stdout + .contains("file is present but drillbit is not running")); + } + + // As should stop. + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STOP).run(); + assertEquals(1, result.returnCode); + assertTrue( + result.stdout.contains("No drillbit to stop because kill -0 of pid")); + } + } + + @Test + public void testStockDaemonWithArg() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + + // As above, but pass an argument. + + { + String propArg = "-Dproperty=value"; + DrillbitRun runner = new DrillbitRun(DrillbitRun.DRILLBIT_START); + runner.withArg(propArg); + RunResult result = runner.start(); + assertEquals(0, result.returnCode); + result.validateArg(propArg); + } + + validateAndCloseDaemon(null); + } + + private void validateAndCloseDaemon(File siteDir) throws IOException { + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STATUS) + .withSite(siteDir) + .run(); + assertEquals(0, result.returnCode); + assertTrue(result.stdout.contains("drillbit is running")); + } + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STOP) + .withSite(siteDir) + .run(); + assertEquals(0, result.returnCode); + } + } + + /** + * The Daemon process creates a pid file. Verify that the DRILL_PID_DIR can be + * set to put the pid file in a custom location. The test is done with the + * site (conf) dir in the default location. + * + * @throws IOException + */ + + @Test + public void testPidDir() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + File pidDir = context.createDir(new File(context.testDir, "pid")); + Map<String, String> drillEnv = new HashMap<>(); + drillEnv.put("DRILL_PID_DIR", pidDir.getAbsolutePath()); + context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv); + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_START) + .withPidDir(pidDir) + .start(); + assertEquals(0, result.returnCode); + assertTrue(result.pidFile.getParentFile().equals(pidDir)); + assertTrue(result.pidFile.exists()); + } + + validateAndCloseDaemon(null); + } + + /** + * Test a custom site directory with the Drill daemon process. The custom + * directory contains a drill-env.sh with a custom option. Verify that that + * option is picked up when starting Drill. + * + * @throws IOException + */ + + @Test + public void testSiteDirWithDaemon() throws IOException { + context.createMockDistrib(); + + File siteDir = new File(context.testDir, "site"); + context.createMockConf(siteDir); + + Map<String, String> drillEnv = new HashMap<>(); + drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "9G"); + context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv); + + // Use the -site (--config) option. + + { + DrillbitRun runner = new DrillbitRun(DrillbitRun.DRILLBIT_START); + runner.withSite(siteDir); + RunResult result = runner.start(); + assertEquals(0, result.returnCode); + result.validateArg("-XX:MaxDirectMemorySize=9G"); + } + + validateAndCloseDaemon(siteDir); + + // Set an env var. + + { + DrillbitRun runner = new DrillbitRun(DrillbitRun.DRILLBIT_START); + runner.addEnv("DRILL_CONF_DIR", siteDir.getAbsolutePath()); + RunResult result = runner.start(); + assertEquals(0, result.returnCode); + result.validateArg("-XX:MaxDirectMemorySize=9G"); + } + + validateAndCloseDaemon(siteDir); + } + + /** + * Launch the Drill daemon using a custom log file location. The config is in + * the default location. + * + * @throws IOException + */ + + @Test + public void testLogDirWithDaemon() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + File logsDir = context.createDir(new File(context.testDir, "logs")); + context.removeDir(new File(context.testDrillHome, "log")); + Map<String, String> drillEnv = new HashMap<>(); + drillEnv.put("DRILL_LOG_DIR", logsDir.getAbsolutePath()); + context.createEnvFile(new File(siteDir, "drill-env.sh"), drillEnv); + + { + DrillbitRun runner = new DrillbitRun(DrillbitRun.DRILLBIT_START); + runner.withLogDir(logsDir); + RunResult result = runner.start(); + assertEquals(0, result.returnCode); + assertNotNull(result.logFile); + assertTrue(result.logFile.getParentFile().equals(logsDir)); + assertTrue(result.logFile.exists()); + assertNotNull(result.outFile); + assertTrue(result.outFile.getParentFile().equals(logsDir)); + assertTrue(result.outFile.exists()); + } + + validateAndCloseDaemon(null); + } + + /** + * Some distributions create symlinks to drillbit.sh in standard locations + * such as /usr/bin. Because drillbit.sh uses its own location to compute + * DRILL_HOME, it must handle symlinks. This test verifies that process. + * + * @throws IOException + */ + + @Test + public void testDrillbitSymlink() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + + File drillbitFile = new File(context.testDrillHome, "bin/drillbit.sh"); + File linksDir = context.createDir(new File(context.testDir, "links")); + File link = new File(linksDir, drillbitFile.getName()); + try { + Files.createSymbolicLink(link.toPath(), drillbitFile.toPath()); + } catch (UnsupportedOperationException e) { + // Well. This is a system without symlinks, so we won't be testing + // syminks here... + + return; + } + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_START).start(); + assertEquals(0, result.returnCode); + assertEquals(result.pidFile.getParentFile(), context.testDrillHome); + } + validateAndCloseDaemon(null); + } + + /** + * Test the restart command of drillbit.sh + * + * @throws IOException + */ + + @Test + public void testRestart() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + + int firstPid; + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_START).start(); + assertEquals(0, result.returnCode); + firstPid = result.getPid(); + } + + // Make sure it is running. + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STATUS) + .withSite(siteDir) + .run(); + assertEquals(0, result.returnCode); + assertTrue(result.stdout.contains("drillbit is running")); + } + + // Restart. Should get new pid. + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_RESTART).start(); + assertEquals(0, result.returnCode); + int secondPid = result.getPid(); + assertNotEquals(firstPid, secondPid); + } + + validateAndCloseDaemon(null); + } + + /** + * Simulate a Drillbit that refuses to die. The stop script wait a while, then + * forces killing. + * + * @throws IOException + */ + + @Test + public void testForcedKill() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + + { + DrillbitRun runner = new DrillbitRun(DrillbitRun.DRILLBIT_START); + runner.addEnv("PRETEND_HUNG", "1"); + RunResult result = runner.start(); + assertEquals(0, result.returnCode); + } + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STATUS) + .preserveLogs() + .run(); + assertEquals(0, result.returnCode); + assertTrue(result.stdout.contains("drillbit is running")); + } + + { + RunResult result = new DrillbitRun(DrillbitRun.DRILLBIT_STOP) + .addEnv("DRILL_STOP_TIMEOUT","5") + .preserveLogs() + .run(); + assertEquals(0, result.returnCode); + assertTrue(result.stdout.contains("drillbit did not complete after")); + } + } + + /** + * Verify the basics of the sqlline script, including the env vars that can be + * customized. Also validate running in embedded mode (using drillbit memory + * and other options.) + * + * @throws IOException + */ + + @Test + public void testSqlline() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + + int stockArgCount; + { + // Out-of-the-box sqlline + + RunResult result = new ScriptRunner("sqlline").run(); + assertEquals(0, result.returnCode); + result.validateJava(); + result.validateClassPath(ScriptUtils.stdCp); + assertTrue(result.containsArgsRegex(ScriptUtils.sqlLineArgs)); + stockArgCount = result.echoArgs.size(); + } + + { + RunResult result = new ScriptRunner("sqlline") + .withArg("arg1") + .withArg("arg2") + .run( ); + assertTrue(result.containsArg("arg1")); + assertTrue(result.containsArg("arg2")); + } + { + // Change drill memory and other drill-specific + // settings: should not affect sqlline + + Map<String, String> drillEnv = new HashMap<>(); + drillEnv.put("DRILL_JAVA_OPTS", "-Dprop=value"); + drillEnv.put("DRILL_HEAP", "5G"); + drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G"); + drillEnv.put("SERVER_LOG_GC", "1"); + drillEnv.put("DRILLBIT_MAX_PERM", "600M"); + drillEnv.put("DRILLBIT_CODE_CACHE_SIZE", "2G"); + RunResult result = new ScriptRunner("sqlline") + .withEnvironment(drillEnv) + .run(); + assertTrue(result.containsArgsRegex(ScriptUtils.sqlLineArgs)); + + // Nothing new should have been added + + assertEquals(stockArgCount, result.echoArgs.size()); + } + { + // Change client memory: should affect sqlline + + Map<String, String> shellEnv = new HashMap<>(); + shellEnv.put("CLIENT_GC_OPTS", "-XX:+UseG1GC"); + shellEnv.put("SQLLINE_JAVA_OPTS", "-XX:MaxPermSize=256M"); + RunResult result = new ScriptRunner("sqlline") + .withEnvironment(shellEnv) + .run(); + assertTrue(result.containsArg("-XX:MaxPermSize=256M")); + assertTrue(result.containsArg("-XX:+UseG1GC")); + } + { + // Change drill memory and other drill-specific + // settings: then set the "magic" variable that says + // that Drill is embedded. The scripts should now use + // the Drillbit options. + + Map<String, String> drillEnv = new HashMap<>(); + drillEnv.put("DRILL_JAVA_OPTS", "-Dprop=value"); + drillEnv.put("DRILL_HEAP", "5G"); + drillEnv.put("DRILL_MAX_DIRECT_MEMORY", "7G"); + drillEnv.put("SERVER_LOG_GC", "1"); + drillEnv.put("DRILLBIT_MAX_PERM", "600M"); + drillEnv.put("DRILLBIT_CODE_CACHE_SIZE", "2G"); + drillEnv.put("DRILL_EMBEDDED", "1"); + RunResult result = new ScriptRunner("sqlline") + .withEnvironment(drillEnv) + .run(); + + String expectedArgs[] = { + "-Dprop=value", + "-Xms5G", "-Xmx5G", + "-XX:MaxDirectMemorySize=7G", + "-XX:ReservedCodeCacheSize=2G", + "-XX:MaxPermSize=600M" + }; + + result.validateArgs(expectedArgs); + assertTrue(result.containsArg("sqlline.SqlLine")); + } + } + + /** + * Verify that the sqlline client works with the --site option by customizing + * items in the site directory. + * + * @throws IOException + */ + + @Test + public void testSqllineSiteDir() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDir, "site"); + context.createMockConf(siteDir); + + // Dummy drill-env.sh to simulate the shipped "example" file + // with some client-specific changes. + + context.writeFile( new File( siteDir, "drill-env.sh" ), + "#!/usr/bin/env bash\n" + + "# Example file\n" + + "export SQLLINE_JAVA_OPTS=\"-XX:MaxPermSize=256M\"\n" + ); + File siteJars = new File(siteDir, "jars"); + context.createDir(siteJars); + context.makeDummyJar(siteJars, "site"); + { + RunResult result = new ScriptRunner("sqlline").withSite(siteDir).run(); + assertEquals(0, result.returnCode); + assertTrue(result.containsArg("-XX:MaxPermSize=256M")); + result.validateClassPath(siteJars.getAbsolutePath() + "/*"); + } + } + + /** + * Tests the three scripts that wrap sqlline for specific purposes: + * <ul> + * <li>drill-conf â Wrapper for sqlline, uses drill config to find Drill. + * Seems this one needs fixing to use a config other than the hard-coded + * $DRILL_HOME/conf location.</li> + * <li>drill-embedded â Starts a drill âembeddedâ in SqlLine, using a local + * ZK.</li> + * <li>drill-localhost â Wrapper for sqlline, uses a local ZK.</li> + * </ul> + * + * Of these, drill-embedded runs an embedded Drillbit and so should use the + * Drillbit memory options. The other two are clients, but with simple default + * options for finding the Drillbit. + * <p> + * Because the scripts are simple wrappers, all we do is verify that the right + * "extra" options are set, not the fundamentals (which were already covered + * in the sqlline tests.) + * + * @throws IOException + */ + + @Test + public void testSqllineWrappers() throws IOException { + context.createMockDistrib(); + File siteDir = new File(context.testDrillHome, "conf"); + context.createMockConf(siteDir); + + { + // drill-conf: just adds a stub JDBC connect string. + + RunResult result = new ScriptRunner("drill-conf") + .withArg("arg1") + .run(); + assertEquals(0, result.returnCode); + result.validateJava(); + result.validateClassPath(ScriptUtils.stdCp); + assertTrue(result.containsArgsRegex(ScriptUtils.sqlLineArgs)); + assertTrue(result.containsArg("-u")); + assertTrue(result.containsArg("jdbc:drill:")); + assertTrue(result.containsArg("arg1")); + } + + { + // drill-localhost: Adds a JDBC connect string to a drillbit + // on the localhost + + RunResult result = new ScriptRunner("drill-localhost") + .withArg("arg1") + .run(); + assertEquals(0, result.returnCode); + result.validateJava(); + result.validateClassPath(ScriptUtils.stdCp); + assertTrue(result.containsArgsRegex(ScriptUtils.sqlLineArgs)); + assertTrue(result.containsArg("-u")); + assertTrue(result.containsArg("jdbc:drill:drillbit=localhost")); + assertTrue(result.containsArg("arg1")); + } + + { + // drill-embedded: Uses drillbit startup options and + // connects to the embedded drillbit. + + RunResult result = new ScriptRunner("drill-embedded") + .withArg("arg1") + .run(); + assertEquals(0, result.returnCode); + result.validateJava(); + result.validateClassPath(ScriptUtils.stdCp); + assertTrue(result.containsArgsRegex(ScriptUtils.sqlLineArgs)); + assertTrue(result.containsArg("-u")); + assertTrue(result.containsArg("jdbc:drill:zk=local")); + assertTrue(result.containsArg("-Xms4G")); + assertTrue(result.containsArg("-XX:MaxDirectMemorySize=8G")); + assertTrue(result.containsArg("arg1")); + } + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestAmRegistration.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestAmRegistration.java b/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestAmRegistration.java new file mode 100644 index 0000000..31c6349 --- /dev/null +++ b/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestAmRegistration.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.zk; + +import static org.junit.Assert.*; + +import org.apache.curator.test.TestingServer; +import org.apache.drill.yarn.appMaster.AMRegistrar.AMRegistrationException; +import org.junit.Test; + +public class TestAmRegistration { + private static final String TEST_CLUSTER_ID = "drillbits"; + private static final String TEST_ZK_ROOT = "drill"; + private static final String TEST_AM_HOST = "localhost"; + private static final int TEST_AM_PORT = 8048; + private static final String TEST_APP_ID = "Application_001"; + + private static final String PROBE_AM_HOST = "somehost"; + private static final String PROBE_APP_ID = "Application_002"; + private static final int PROBE_AM_PORT = 8049; + private static final String PROBE_CLUSTER_ID = "second"; + private static final String PROBE_ZK_ROOT = "myRoot"; + + @Test + public void testAMRegistry() throws Exception { + try (TestingServer server = new TestingServer()) { + server.start(); + String connStr = server.getConnectString(); + ZKClusterCoordinatorDriver driver = new ZKClusterCoordinatorDriver() + .setConnect(connStr, TEST_ZK_ROOT, TEST_CLUSTER_ID).build(); + + // Register an AM using the above. + + driver.register(TEST_AM_HOST, TEST_AM_PORT, TEST_APP_ID); + + // Simulate a second AM for the same cluster. + + { + ZKClusterCoordinatorDriver driver2 = new ZKClusterCoordinatorDriver() + .setConnect(connStr, TEST_ZK_ROOT, TEST_CLUSTER_ID).build(); + + // Register an AM on the same (root, cluster id). + + try { + driver.register(PROBE_AM_HOST, PROBE_AM_PORT, PROBE_APP_ID); + fail(); + } catch (AMRegistrationException e) { + String msg = e.getMessage(); + assertTrue(msg.contains("Application Master already exists")); + assertTrue( + msg.contains(" " + TEST_ZK_ROOT + "/" + TEST_CLUSTER_ID + " ")); + assertTrue(msg.contains(" host: " + TEST_AM_HOST)); + assertTrue(msg.contains(" Application ID: " + TEST_APP_ID)); + } + + driver2.close(); + } + + { + ZKClusterCoordinatorDriver driver2 = new ZKClusterCoordinatorDriver() + .setConnect(connStr, TEST_ZK_ROOT, PROBE_CLUSTER_ID).build(); + + // Register an AM on a different cluster id, same root. + + try { + driver2.register(PROBE_AM_HOST, PROBE_AM_PORT, PROBE_APP_ID); + } catch (AMRegistrationException e) { + fail("Registration should be OK"); + } + + driver2.close(); + } + + { + ZKClusterCoordinatorDriver driver2 = new ZKClusterCoordinatorDriver() + .setConnect(connStr, PROBE_ZK_ROOT, TEST_CLUSTER_ID).build(); + + // Register an AM on a different root. + + try { + driver2.register(PROBE_AM_HOST, PROBE_AM_PORT, PROBE_APP_ID); + } catch (AMRegistrationException e) { + fail("Registration should be OK"); + } + + driver2.close(); + } + + // First AM exits. + + driver.close(); + + { + // Should be able to register an AM for the same cluster. + + ZKClusterCoordinatorDriver driver2 = new ZKClusterCoordinatorDriver() + .setConnect(connStr, TEST_ZK_ROOT, TEST_CLUSTER_ID).build(); + + // Register an AM on the same (root, cluster id). + + try { + driver2.register(PROBE_AM_HOST, PROBE_AM_PORT, PROBE_APP_ID); + } catch (AMRegistrationException e) { + fail("Registration should be OK"); + } + + driver2.close(); + } + + server.stop(); + } + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestZkRegistry.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestZkRegistry.java b/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestZkRegistry.java new file mode 100644 index 0000000..b6f6540 --- /dev/null +++ b/drill-yarn/src/test/java/org/apache/drill/yarn/zk/TestZkRegistry.java @@ -0,0 +1,459 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.zk; + +import static org.junit.Assert.*; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryNTimes; +import org.apache.curator.test.TestingServer; +import org.apache.curator.x.discovery.ServiceInstance; +import org.apache.drill.exec.coord.DrillServiceInstanceHelper; +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; +import org.apache.drill.exec.work.foreman.DrillbitStatusListener; +import org.apache.drill.yarn.appMaster.EventContext; +import org.apache.drill.yarn.appMaster.RegistryHandler; +import org.apache.drill.yarn.appMaster.Task; +import org.apache.drill.yarn.appMaster.TaskLifecycleListener.Event; +import org.apache.drill.yarn.zk.ZKRegistry.DrillbitTracker; +import org.junit.Test; + +/** + * Tests for the AM version of the cluster coordinator. The AM version has no + * dependencies on the DoY config system or other systems, making it easy to + * test in isolation using the Curator-provided test server. + */ + +public class TestZkRegistry { + private static final String BARNEY_HOST = "barney"; + private static final String WILMA_HOST = "wilma"; + private static final String TEST_HOST = "host"; + private static final String FRED_HOST = "fred"; + public static final int TEST_USER_PORT = 123; + public static final int TEST_CONTROL_PORT = 456; + public static final int TEST_DATA_PORT = 789; + public static final String ZK_ROOT = "test-root"; + public static final String CLUSTER_ID = "test-cluster"; + + /** + * Validate that the key format used for endpoint is the same as that + * generated for hosts coming from YARN. + */ + + @Test + public void testFormat() { + DrillbitEndpoint dbe = makeEndpoint(TEST_HOST); + assertEquals(makeKey(TEST_HOST), ZKClusterCoordinatorDriver.asString(dbe)); + + ZKClusterCoordinatorDriver driver = new ZKClusterCoordinatorDriver() + .setPorts(123, 456, 789); + assertEquals(makeKey(TEST_HOST), driver.toKey(TEST_HOST)); + + // Internal default ports (used mostly for testing.) + + driver = new ZKClusterCoordinatorDriver(); + assertEquals("fred:31010:31011:31012", driver.toKey(FRED_HOST)); + } + + public static String makeKey(String host) { + return host + ":" + TEST_USER_PORT + ":" + TEST_CONTROL_PORT + ":" + + TEST_DATA_PORT; + } + + /** + * Basic setup: start a ZK and verify that the initial endpoint list is empty. + * Also validates the basics of the test setup (mock server, etc.) + * + * @throws Exception + */ + + @Test + public void testBasics() throws Exception { + try (TestingServer server = new TestingServer()) { + server.start(); + String connStr = server.getConnectString(); + ZKClusterCoordinatorDriver driver = new ZKClusterCoordinatorDriver() + .setConnect(connStr, "drill", "drillbits").build(); + assertTrue(driver.getInitialEndpoints().isEmpty()); + driver.close(); + server.stop(); + } + } + + private class TestDrillbitStatusListener implements DrillbitStatusListener { + protected Set<DrillbitEndpoint> added; + protected Set<DrillbitEndpoint> removed; + + @Override + public void drillbitUnregistered( + Set<DrillbitEndpoint> unregisteredDrillbits) { + removed = unregisteredDrillbits; + } + + @Override + public void drillbitRegistered(Set<DrillbitEndpoint> registeredDrillbits) { + added = registeredDrillbits; + } + + public void clear() { + added = null; + removed = null; + } + } + + /** + * Test a typical life cycle: existing Drillbit on AM start, add a Drilbit + * (simulates a drillbit starting), and remove a drillbit (simulates a + * Drillbit ending). + * + * @throws Exception + */ + + @Test + public void testCycle() throws Exception { + TestingServer server = new TestingServer(); + server.start(); + String connStr = server.getConnectString(); + + CuratorFramework probeZk = connectToZk(connStr); + addDrillbit(probeZk, FRED_HOST); + + ZKClusterCoordinatorDriver driver = new ZKClusterCoordinatorDriver() + .setConnect(connStr, ZK_ROOT, CLUSTER_ID).build(); + List<DrillbitEndpoint> bits = driver.getInitialEndpoints(); + assertEquals(1, bits.size()); + assertEquals(makeKey(FRED_HOST), + ZKClusterCoordinatorDriver.asString(bits.get(0))); + + TestDrillbitStatusListener listener = new TestDrillbitStatusListener(); + driver.addDrillbitListener(listener); + + addDrillbit(probeZk, WILMA_HOST); + Thread.sleep(50); + assertNull(listener.removed); + assertNotNull(listener.added); + assertEquals(1, listener.added.size()); + for (DrillbitEndpoint dbe : listener.added) { + assertEquals(makeKey(WILMA_HOST), + ZKClusterCoordinatorDriver.asString(dbe)); + } + + listener.clear(); + removeDrillbit(probeZk, FRED_HOST); + Thread.sleep(50); + assertNull(listener.added); + assertNotNull(listener.removed); + assertEquals(1, listener.removed.size()); + for (DrillbitEndpoint dbe : listener.removed) { + assertEquals(makeKey(FRED_HOST), + ZKClusterCoordinatorDriver.asString(dbe)); + } + + probeZk.close(); + driver.close(); + server.stop(); + server.close(); + } + + /** + * Make a Drill endpoint using the hard-coded test ports and the given host + * name. + * + * @param host + * @return + */ + + private DrillbitEndpoint makeEndpoint(String host) { + return DrillbitEndpoint.newBuilder().setAddress(host) + .setControlPort(TEST_CONTROL_PORT).setDataPort(TEST_DATA_PORT) + .setUserPort(TEST_USER_PORT).build(); + } + + /** + * Pretend to be a Drillbit creating its ZK entry. Real Drillbits use a GUID + * as the key, but we just use the host name, which is good enough for our + * purposes here. + * + * @param zk + * @param host + * @throws Exception + */ + + private void addDrillbit(CuratorFramework zk, String host) throws Exception { + DrillbitEndpoint dbe = makeEndpoint(host); + ServiceInstance<DrillbitEndpoint> si = ServiceInstance + .<DrillbitEndpoint> builder().name(CLUSTER_ID).payload(dbe).build(); + byte data[] = DrillServiceInstanceHelper.SERIALIZER.serialize(si); + zk.create().forPath("/" + host, data); + } + + private void removeDrillbit(CuratorFramework zk, String host) + throws Exception { + zk.delete().forPath("/" + host); + } + + /** + * Connect to the test ZK for the simulated Drillbit side of the test. (The AM + * side of the test uses the actual AM code, which is what we're testing + * here...) + * + * @param connectString + * @return + */ + + public static CuratorFramework connectToZk(String connectString) { + CuratorFramework client = CuratorFrameworkFactory.builder() + .namespace(ZK_ROOT + "/" + CLUSTER_ID).connectString(connectString) + .retryPolicy(new RetryNTimes(3, 1000)).build(); + client.start(); + return client; + } + + private static class TestRegistryHandler implements RegistryHandler { + String reserved; + String released; + Task start; + Task end; + + public void clear() { + reserved = null; + released = null; + start = null; + end = null; + } + + @Override + public void reserveHost(String hostName) { + assertNull(reserved); + reserved = hostName; + } + + @Override + public void releaseHost(String hostName) { + assertNull(released); + released = hostName; + } + + @Override + public void startAck(Task task, String propertyKey, Object value) { + start = task; + } + + @Override + public void completionAck(Task task, String endpointProperty) { + end = task; + } + + @Override + public void registryDown() { + // TODO Auto-generated method stub + + } + } + + public static class TestTask extends Task { + private String host; + + public TestTask(String host) { + super(null, null); + this.host = host; + } + + @Override + public String getHostName() { + return host; + } + + @Override + public void resetTrackingState() { + trackingState = TrackingState.NEW; + } + } + + @Test + public void testZKRegistry() throws Exception { + TestingServer server = new TestingServer(); + server.start(); + String connStr = server.getConnectString(); + + CuratorFramework probeZk = connectToZk(connStr); + addDrillbit(probeZk, FRED_HOST); + + ZKClusterCoordinatorDriver driver = new ZKClusterCoordinatorDriver() + .setConnect(connStr, ZK_ROOT, CLUSTER_ID) + .setPorts(TEST_USER_PORT, TEST_CONTROL_PORT, TEST_DATA_PORT).build(); + ZKRegistry registry = new ZKRegistry(driver); + TestRegistryHandler handler = new TestRegistryHandler(); + registry.start(handler); + + // We started with one "stray" drillbit that will be reported as unmanaged. + + assertEquals(FRED_HOST, handler.reserved); + List<String> unmanaged = registry.listUnmanagedDrillits(); + assertEquals(1, unmanaged.size()); + String fredsKey = makeKey(FRED_HOST); + assertEquals(fredsKey, unmanaged.get(0)); + Map<String, DrillbitTracker> trackers = registry.getRegistryForTesting(); + assertEquals(1, trackers.size()); + assertTrue(trackers.containsKey(fredsKey)); + DrillbitTracker fredsTracker = trackers.get(fredsKey); + assertEquals(fredsKey, fredsTracker.key); + assertEquals(DrillbitTracker.State.UNMANAGED, fredsTracker.state); + assertNull(fredsTracker.task); + assertEquals(fredsKey, + ZKClusterCoordinatorDriver.asString(fredsTracker.endpoint)); + + // The handler should have been told about the initial stray. + + assertEquals(FRED_HOST, handler.reserved); + + // Pretend to start a new Drillbit. + + Task wilmasTask = new TestTask(WILMA_HOST); + EventContext context = new EventContext(wilmasTask); + + // Registry ignores the created event. + + registry.stateChange(Event.CREATED, context); + assertEquals(1, registry.getRegistryForTesting().size()); + + // But, does care about the allocated event. + + registry.stateChange(Event.ALLOCATED, context); + assertEquals(2, registry.getRegistryForTesting().size()); + String wilmasKey = makeKey(WILMA_HOST); + DrillbitTracker wilmasTracker = registry.getRegistryForTesting() + .get(wilmasKey); + assertNotNull(wilmasTracker); + assertEquals(wilmasTask, wilmasTracker.task); + assertNull(wilmasTracker.endpoint); + assertEquals(wilmasKey, wilmasTracker.key); + assertEquals(DrillbitTracker.State.NEW, wilmasTracker.state); + handler.clear(); + + // Time goes on. The Drillbit starts and registers itself. + + addDrillbit(probeZk, WILMA_HOST); + Thread.sleep(100); + assertEquals(wilmasTask, handler.start); + assertEquals(DrillbitTracker.State.REGISTERED, wilmasTracker.state); + assertEquals(handler.start, wilmasTask); + + // Create another task: Barney + + Task barneysTask = new TestTask(BARNEY_HOST); + context = new EventContext(barneysTask); + registry.stateChange(Event.CREATED, context); + + // Start Barney, but assume a latency in Yarn, but not ZK. + // We get the ZK registration before the YARN launch confirmation. + + handler.clear(); + addDrillbit(probeZk, BARNEY_HOST); + Thread.sleep(100); + assertEquals(BARNEY_HOST, handler.reserved); + String barneysKey = makeKey(BARNEY_HOST); + DrillbitTracker barneysTracker = registry.getRegistryForTesting() + .get(barneysKey); + assertNotNull(barneysTracker); + assertEquals(DrillbitTracker.State.UNMANAGED, barneysTracker.state); + assertNull(barneysTracker.task); + assertEquals(2, registry.listUnmanagedDrillits().size()); + + handler.clear(); + registry.stateChange(Event.ALLOCATED, context); + assertEquals(DrillbitTracker.State.REGISTERED, barneysTracker.state); + assertEquals(handler.start, barneysTask); + assertEquals(barneysTask, barneysTracker.task); + assertEquals(1, registry.listUnmanagedDrillits().size()); + + // Barney is having problems, it it drops out of ZK. + + handler.clear(); + removeDrillbit(probeZk, BARNEY_HOST); + Thread.sleep(100); + assertEquals(barneysTask, handler.end); + assertEquals(DrillbitTracker.State.DEREGISTERED, barneysTracker.state); + + // Barney comes alive (presumably before the controller gives up and kills + // the Drillbit.) + + handler.clear(); + addDrillbit(probeZk, BARNEY_HOST); + Thread.sleep(100); + assertEquals(barneysTask, handler.start); + assertEquals(DrillbitTracker.State.REGISTERED, barneysTracker.state); + + // Barney is killed by the controller. + // ZK entry drops. Tracker is removed, controller is notified. + + handler.clear(); + removeDrillbit(probeZk, BARNEY_HOST); + Thread.sleep(100); + assertNotNull(registry.getRegistryForTesting().get(barneysKey)); + assertEquals(barneysTask, handler.end); + + // The controller tells the registry to stop tracking the Drillbit. + + handler.clear(); + registry.stateChange(Event.ENDED, context); + assertNull(handler.end); + assertNull(registry.getRegistryForTesting().get(barneysKey)); + + // The stray drillbit deregisters from ZK. The tracker is removed. + + handler.clear(); + removeDrillbit(probeZk, FRED_HOST); + Thread.sleep(100); + assertNull(registry.getRegistryForTesting().get(fredsKey)); + assertNull(handler.end); + assertEquals(FRED_HOST, handler.released); + + // Wilma is killed by the controller. + + handler.clear(); + removeDrillbit(probeZk, WILMA_HOST); + Thread.sleep(100); + assertEquals(wilmasTask, handler.end); + assertNull(handler.released); + assertEquals(DrillbitTracker.State.DEREGISTERED, wilmasTracker.state); + assertNotNull(registry.getRegistryForTesting().get(wilmasKey)); + + handler.clear(); + context = new EventContext(wilmasTask); + registry.stateChange(Event.ENDED, context); + assertNull(registry.getRegistryForTesting().get(wilmasKey)); + assertNull(handler.released); + assertNull(handler.end); + + // All drillbits should be gone. + + assertTrue(registry.getRegistryForTesting().isEmpty()); + + probeZk.close(); + driver.close(); + server.stop(); + server.close(); + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/resources/doy-test-logback.xml ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/resources/doy-test-logback.xml b/drill-yarn/src/test/resources/doy-test-logback.xml new file mode 100644 index 0000000..b00ff5f --- /dev/null +++ b/drill-yarn/src/test/resources/doy-test-logback.xml @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> +<configuration> + +<!-- + <appender name="SOCKET" class="de.huxhorn.lilith.logback.appender.ClassicMultiplexSocketAppender"> + <Compressing>true</Compressing> + <ReconnectionDelay>10000</ReconnectionDelay> + <IncludeCallerData>true</IncludeCallerData> + <RemoteHosts>${LILITH_HOSTNAME:-localhost}</RemoteHosts> + </appender> --> + + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <!-- encoders are assigned the type + ch.qos.logback.classic.encoder.PatternLayoutEncoder by default --> + <encoder> + <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern> + </encoder> + </appender> + +<!-- + <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>/logs/test-common.log</file> + <encoder> + <pattern>%date %level [%thread] %logger{10} [%file:%line] %msg%n</pattern> + </encoder> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>/logs/test-common.%d{yyyy-MM-dd}.log</fileNamePattern> + <maxHistory>30</maxHistory> + </rollingPolicy> + </appender> + --> + <logger name="org.apache.drill" additivity="false"> + <level value="error" /> + <!-- <appender-ref ref="SOCKET" /> --> + <appender-ref ref="STDOUT" /> +<!-- <appender-ref ref="FILE" /> --> + </logger> + + <root> + <level value="error" /> + <!-- <appender-ref ref="SOCKET" /> --> + <appender-ref ref="STDOUT" /> +<!-- <appender-ref ref="FILE" /> --> + </root> + +</configuration> http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/resources/second-test-config.conf ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/resources/second-test-config.conf b/drill-yarn/src/test/resources/second-test-config.conf new file mode 100644 index 0000000..11c180f --- /dev/null +++ b/drill-yarn/src/test/resources/second-test-config.conf @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file contains a test set of configuration settings used by the +# TestConfig unit test. It has a benign name to prevent any accidental +# inclusion into production code. The test class makes a copy of this +# file with the desired name for the life of the test run only. + +drill.yarn: { + app-name: "My-App" + + drill-install: { + localize: false + drill-home: "/config/drill/home" + site-dir: "/config/drill/site" + client-path: "/foo/bar/drill-archive.tar.gz" + } + + yarn: { + queue: "my-queue" + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/resources/test-doy-config.conf ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/resources/test-doy-config.conf b/drill-yarn/src/test/resources/test-doy-config.conf new file mode 100644 index 0000000..b588530 --- /dev/null +++ b/drill-yarn/src/test/resources/test-doy-config.conf @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file contains a test set of configuration settings used by the +# TestConfig unit test. It has a benign name to prevent any accidental +# inclusion into production code. The test class makes a copy of this +# file with the desired name for the life of the test run only. + +drill.yarn: { + app-name: "My-App" + + drill-install: { + drill-home: "/config/drill/home" + client-path: "/foo/bar/drill-archive.tar.gz" + } + + yarn: { + queue: "my-queue" + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/resources/test-doy-distrib.conf ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/resources/test-doy-distrib.conf b/drill-yarn/src/test/resources/test-doy-distrib.conf new file mode 100644 index 0000000..ff73ef3 --- /dev/null +++ b/drill-yarn/src/test/resources/test-doy-distrib.conf @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file contains a test set of configuration settings used by the +# TestConfig unit test. It has a benign name to prevent any accidental +# inclusion into production code. The test class makes a copy of this +# file with the desired name for the life of the test run only. + +drill.yarn: { + app-name: "config-app-name" + + dfs: { + app-dir: "/opt/drill" + } + yarn: { + queue: "distrib-queue" + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/resources/third-test-config.conf ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/resources/third-test-config.conf b/drill-yarn/src/test/resources/third-test-config.conf new file mode 100644 index 0000000..f585e22 --- /dev/null +++ b/drill-yarn/src/test/resources/third-test-config.conf @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file contains a test set of configuration settings used by the +# TestConfig unit test. It has a benign name to prevent any accidental +# inclusion into production code. The test class makes a copy of this +# file with the desired name for the life of the test run only. + +drill.yarn: { + app-name: "My-App" + + drill-install: { + localize: false + drill-home: "/config/drill/home" + } + + yarn: { + queue: "my-queue" + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/resources/wrapper.sh ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/resources/wrapper.sh b/drill-yarn/src/test/resources/wrapper.sh new file mode 100644 index 0000000..f436eb5 --- /dev/null +++ b/drill-yarn/src/test/resources/wrapper.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Drill Stdout Message" +echo "Stderr Message" 1>&2 + +output="$DRILL_HOME/../output.txt" +while [[ $# > 0 ]] +do + if [[ $1 =~ ^-Dlog.path= ]]; then + thelog=${1/-Dlog.path=//} + echo "Drill Log Message" >> $thelog + fi + echo $1 >> $output + shift +done + +function clean_up { + echo "Received SIGTERM" + if [ "$PRETEND_HUNG" == "1" ]; then + echo "Pretending to be hung." + else + echo "Exiting" + if [ "$PRETEND_FAIL" == "1" ]; then + exit 55 + else + exit 0 + fi + fi +} + +trap clean_up SIGTERM + +if [ "$KEEP_RUNNING" == "1" ]; then + while [[ 1 > 0 ]] + do + sleep 1 + done +fi http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2c1f5fc..a163270 100644 --- a/pom.xml +++ b/pom.xml @@ -1555,6 +1555,62 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + <version>${hadoop.version}</version> + <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-client</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-xc</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + </exclusion> + + <!-- Exclude the following to prevent enforcer failures. YARN uses them, + but Drill forbids them. --> + + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> <version>${hbase.version}</version> @@ -2143,6 +2199,62 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + <version>${hadoop.version}</version> + <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-client</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-xc</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + </exclusion> + + <!-- Exclude the following to prevent enforcer failures. YARN uses them, + but Drill forbids them. --> + + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.antlr</groupId> <artifactId>antlr4-runtime</artifactId> <version>4.5</version> @@ -2506,6 +2618,62 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + <version>2.3.0-cdh5.0.3</version> + <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-client</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-xc</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + </exclusion> + + <!-- Exclude the following to prevent enforcer failures. YARN uses them, + but Drill forbids them. --> + + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> </dependencies> </dependencyManagement> <repositories> @@ -2580,6 +2748,62 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + <version>2.4.0.2.1.3.0-563</version> + <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-json</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-client</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-xc</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + </exclusion> + + <!-- Exclude the following to prevent enforcer failures. YARN uses them, + but Drill forbids them. --> + + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> </dependencies> </dependencyManagement> <repositories> @@ -2723,6 +2947,7 @@ <module>common</module> <module>logical</module> <module>exec</module> + <module>drill-yarn</module> <module>distribution</module> </modules> </project>
