http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/manage.ftl ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/resources/drill-am/manage.ftl b/drill-yarn/src/main/resources/drill-am/manage.ftl new file mode 100644 index 0000000..682530c --- /dev/null +++ b/drill-yarn/src/main/resources/drill-am/manage.ftl @@ -0,0 +1,78 @@ +<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + +<#include "*/generic.ftl"> +<#macro page_head> +</#macro> + +<#macro page_body> + <h4>Manage Drill Cluster</h4> + + Current Status: ${model.getLiveCount( )} + <#if model.getLiveCount( ) == 1 >Drillbit is + <#else>Drillbits are + </#if>running. + <p> + Free YARN nodes: Approximately ${model.getFreeNodeCount( )} + <p><p> + + <table class="table table-hover" style="width: auto;"> + <#-- Removed per user feedback. (Kept in REST API as client needs them. + <tr><td style="vertical-align: middle;"> + <form action="/resize" method="POST" class="form-inline" role="form"> + <div class="form-group"> + <input hidden name="type" value="grow"> + <label for="add">Add</label> + <input type="text" name="n" size="6" id="add" class="form-control" + placeholder="+n" style="padding: 0 1em; margin: 0 1em;"/> + drillbits. + <button type="submit" class="btn btn-primary" style="margin: 0 1em;">Go</button> + </div> + </form> + </td></tr> + <tr><td> + <form action="/resize" method="POST" class="form-inline" role="form"> + <div class="form-group"> + <input hidden name="type" value="shrink"> + <label for="shrink">Remove</label> + <input type="text" name="n" size="6" id="shrink" class="form-control" + placeholder="-n" style="padding: 0 1em; margin: 0 1em;"/> + drillbits. + <button type="submit" class="btn btn-primary" style="margin: 0 1em;">Go</button> + </div> + </form> + </td></tr> + --> + <tr><td> + <form action="/resize" method="POST" class="form-inline" role="form"> + <div class="form-group"> + <input hidden name="type" value="resize"> + <label for="resize">Resize to</label> + <input type="text" name="n" id="resize" size="6" + placeholder="Size" class="form-control" style="padding: 0 1em; margin: 0 1em;"/> + drillbits. + <button type="submit" class="btn btn-primary" style="margin: 0 1em;">Go</button> + </div> + </form> + </td></tr> + <tr><td> + <form action="/stop" method="GET" class="form-inline" role="form"> + <div class="form-group"> + <label for="stop">Stop</label> the Drill cluster. + <button type="submit" id="stop" class="btn btn-primary" style="margin: 0 1em;">Go</button> + </div> + </form> + </td></tr> + </table> + +</#macro> + +<@page_html/>
http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/redirect.ftl ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/resources/drill-am/redirect.ftl b/drill-yarn/src/main/resources/drill-am/redirect.ftl new file mode 100644 index 0000000..b38d10c --- /dev/null +++ b/drill-yarn/src/main/resources/drill-am/redirect.ftl @@ -0,0 +1,33 @@ +<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Apache Drill - Application Master</title> + <META http-equiv="refresh" content="0;URL=${amLink}"> + <style> + body { font-family: sans-serif; + text-align: center; } + </style> + </head> + <body> + <h3>YARN Application Master – ${clusterName}</h3> + <h4>Redirect</h4> + The Drill Application Master UI does not work correctly inside the proxy page + provided by YARN. + <p>Click + <a href="${amLink}">here</a> to go to the Application Master directly + if this page does not automatically redirect you. + </body> +</html> + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/shrink-warning.ftl ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/resources/drill-am/shrink-warning.ftl b/drill-yarn/src/main/resources/drill-am/shrink-warning.ftl new file mode 100644 index 0000000..dded46a --- /dev/null +++ b/drill-yarn/src/main/resources/drill-am/shrink-warning.ftl @@ -0,0 +1,58 @@ +<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + +<#include "*/generic.ftl"> +<#macro page_head> +</#macro> + +<#macro page_body> + <h4><#if model.isStop( )> + Confirm Cluster Shutdown + <#else> + Confirm Stopping of Drillbits + </#if></h4> + + <div class="alert alert-danger"> + <strong>Warning!</strong> You have requested to + <#if model.isStop()> + stop the Drill cluster. + <#elseif model.isCancel()> + cancel Drillbit ${model.getId()}. + <#elseif model.isKill()> + kill Drillbit ${model.getId()}. + <#else> + remove ${model.getCount( )} + <#if model.getCount() == 1>Drillbit<#else>Drillbits</#if>. + </#if> + <#if ! model.isCancel()> + In this version of Drill, stopping Drillbits will + cause in-flight queries to fail. + </#if> + </div> + <#if model.isStop( )> + <form method="POST" action="/stop"> + <#elseif model.isCancel( )> + <form method="POST" action="/cancel?id=${model.getId( )}"> + <#elseif model.isKill( )> + <form method="POST" action="/cancel?id=${model.getId( )}"> + <#else> + <form method="POST" action="/resize"> + </#if> + <#if model.isShrink( )> + <input type="hidden" name="n" value="${model.getCount( )}"> + <input type="hidden" name="type" value="force-shrink"> + </#if> + <input type="submit" value="Confirm"> or + <a href="/">Cancel</a>. + </form> +</#macro> + +<@page_html/> http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/static/css/drill-am.css ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/resources/drill-am/static/css/drill-am.css b/drill-yarn/src/main/resources/drill-am/static/css/drill-am.css new file mode 100644 index 0000000..a58fad1 --- /dev/null +++ b/drill-yarn/src/main/resources/drill-am/static/css/drill-am.css @@ -0,0 +1,20 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. */ + +body { + padding-top: 50px; +} +.drill-am { + padding: 0 15px 20px 15px; +} +h3 { + text-align: center; +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/static/img/apache-drill-logo.png ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/resources/drill-am/static/img/apache-drill-logo.png b/drill-yarn/src/main/resources/drill-am/static/img/apache-drill-logo.png new file mode 100644 index 0000000..bce39c0 Binary files /dev/null and b/drill-yarn/src/main/resources/drill-am/static/img/apache-drill-logo.png differ http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/static/img/drill.ico ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/resources/drill-am/static/img/drill.ico b/drill-yarn/src/main/resources/drill-am/static/img/drill.ico new file mode 100644 index 0000000..0f9654e Binary files /dev/null and b/drill-yarn/src/main/resources/drill-am/static/img/drill.ico differ http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/drill-am/tasks.ftl ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/resources/drill-am/tasks.ftl b/drill-yarn/src/main/resources/drill-am/tasks.ftl new file mode 100644 index 0000000..c6725b1 --- /dev/null +++ b/drill-yarn/src/main/resources/drill-am/tasks.ftl @@ -0,0 +1,113 @@ +<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + +<#include "*/generic.ftl"> +<#macro page_head> + <meta http-equiv="refresh" content="${refreshSecs}" > +</#macro> + +<#macro page_body> + <h4>Drillbit Status</h4> + <p> + + <#if model.hasTasks( ) > + <div class="table-responsive"> + <table class="table table-hover"> + <tr> + <th><span data-toggle="tooltip" title="Internal AM ID for the Drillbit.">ID</span></th> + <th><span data-toggle="tooltip" title="Cluster group from config file">Group</span></th> + <th><span data-toggle="tooltip" + title="Host name or IP running the Drillbit and ink to Drillbit web UI."> + Host</span></th> + <th><span data-toggle="tooltip" title="State of the Drillbit process, hover for details.">State</span></th> + <th><span data-toggle="tooltip" title="ZooKeeper tracking state for the Drillbit, hover for details.">ZK State</span></th> + <th><span data-toggle="tooltip" + title="YARN Container allocated to the Drillbit and link to the YARN Node Manager container UI."> + Container</span></th> + <th><span data-toggle="tooltip" title="Memory granted by YARN to the Drillbit.">Memory (MB)</span></th> + <th><span data-toggle="tooltip" title="Virtual cores granted by YARN to the Drillbit.">Virtual Cores</span></th> + <#if showDisks > + <th><span data-toggle="tooltip" title="Disk resources granted by YARN to the Drillbit.">Disks</span></th> + </#if> + <th><span data-toggle="tooltip" title="Start time in the AM server time zone.">Start Time</span></th> + </th> + <#list tasks as task> + <tr> + <td><b>${task.getTaskId( )}</b></td> + <td>${task.getGroupName( )}</td> + <td> + <#if task.isLive( )> + <a href="${task.getLink( )}" data-toggle="tooltip" title="Link to the Drillbit Web UI"></#if> + ${task.getHost( )} + <#if task.isLive( )></a></#if> + </td> + <td><span data-toggle="tooltip" title="${task.getStateHint( )}">${task.getState( )}</span> + <#if task.isCancelled( )><br/>(Cancelled)</#if> + <#if task.isCancellable( )> + <a href="/cancel?id=${task.getTaskId( )}" data-toggle="tooltip" title="Kill this Drillbit">[x]</a> + </#if> + </td> + <td><span data-toggle="tooltip" title="${task.getTrackingStateHint( )}">${task.getTrackingState( )}</span></td> + <td><#if task.hasContainer( )> + <a href="${task.getNmLink( )}" data-toggle="tooltip" title="Node Manager UI for Drillbit container">${task.getContainerId()}</a> + <#else> </#if></td> + <td>${task.getMemory( )}</td> + <td>${task.getVcores( )}</td> + <#if showDisks > + <td>${task.getDisks( )}</td> + </#if> + <td>${task.getStartTime( )}</td> + </tr> + </#list> + </table> + <#else> + <div class="alert alert-danger"> + No drillbits are running. + </div> + </#if> + <#if model.hasUnmanagedDrillbits( ) > + <hr> + <div class="alert alert-danger"> + <strong>Warning:</strong> ZooKeeper reports that + ${model.getUnmanagedDrillbitCount( )} Drillbit(s) are running that were not + started by the YARN Application Master. Perhaps they were started manually. + </div> + <table class="table table-hover" style="width: auto;"> + <tr><th>Drillbit Host</th><th>Ports</th></tr> + <#list strays as stray > + <tr> + <td>${stray.getHost( )}</td> + <td>${stray.getPorts( )}</td> + </tr> + </#list> + </table> + </#if> + <#if model.hasBlacklist( ) > + <hr> + <div class="alert alert-danger"> + <strong>Warning:</strong> + ${model.getBlacklistCount( )} nodes have been black-listed due to + repeated Drillbit launch failures. Perhaps the nodes or Drill are + improperly configured. + </div> + <table class="table table-hover" style="width: auto;"> + <tr><th>Blacklisted Host</th></tr> + <#list blacklist as node > + <tr> + <td>${node}</td> + </tr> + </#list> + </table> + </#if> + </div> +</#macro> + +<@page_html/> http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/main/resources/org/apache/drill/yarn/core/drill-on-yarn-defaults.conf ---------------------------------------------------------------------- diff --git a/drill-yarn/src/main/resources/org/apache/drill/yarn/core/drill-on-yarn-defaults.conf b/drill-yarn/src/main/resources/org/apache/drill/yarn/core/drill-on-yarn-defaults.conf new file mode 100644 index 0000000..bda8152 --- /dev/null +++ b/drill-yarn/src/main/resources/org/apache/drill/yarn/core/drill-on-yarn-defaults.conf @@ -0,0 +1,275 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- +# Configuration for the Drill-on-YARN feature. +# See drill-on-yarn-example.conf for details. +# Some properties are private; the are meant for use by the Drill +# developers. Those are marked as private here, and are explained here +# rather than in the example config. + +drill.yarn: { + app-name: "Drill-on-YARN" + + # Settings here support a default single-node cluster on the local host, + # using the default HDFS connection obtained from the Hadoop config files. + + dfs: { + connection: "" + app-dir: "/user/drill" + } + + drill-install: { + site-dir: "" + client-path: "" + dir-name: "<base>" + + # Drill-on-YARN uses localization by default; it is the simplest and + # most reliable for production systems. However, localization can be + # disabled for development, testing or other special needs. + + localize: true + + # Localized runs typically provide a site directory. For various reasons, + # some users may wish to avoid using a site directory and instead just + # use their own $DRILL_HOME/conf directory as the site directory. In + # this case, $DRILL_HOME/conf must contain all site-specific content, + # including jars. This setting says to treat $DRILL_HOME/conf as the + # site directory. Disable this option if the Drill archive is rebuilt + # to contain the $DRILL_HOME/conf files. This flag is also automatically + # assumed if the Drill archive is *inside* $DRILL_HOME. + + conf-as-site: true + + # When localization is disabled, the path to Drill software defaults + # to the same location as on the client node. However, if the path + # on worker nodes is different, set the actual path here. + + drill-home: "" + + # Drill provides the option to store site-specific configuration outside + # of the Drill home directory. The site directory contains configuration + # files and custom Java code (in the jars subdirectory.) The site dir + # normally is localized by the client. However, in a non-localized + # run, use this setting to identify the location of the site directory + # on each worker node if the location is different than the node + # that runs the client. + + site-dir: "" + + # Key used for the Drill archive file in the AM launch config. + # Not usually changed by users. + + drill-key: "drill" + site-key: "site" + + # Set the Java java.library.path option to files that pre-exist on + # each Drillbit node. (This is not for libraries that are distributed + # by YARN.) + + library-path: "" + } + + yarn: { + queue: "default" + priority: 1 + user: "" + } + + hadoop: { + home: "" + class-path: "" + hbase-class-path: "" + } + + client: { + + # Time (in secs) between poll attempts on start and stop. + + poll-sec: 1 + + # Maximum time to wait when starting the AM before asking the + # user to check status later. + + start-wait-sec: 60 + + # Maximum time to wait when stopping the AM before skipping + # the confirmation step. + + stop-wait-sec: 60 + } + + am: { + memory-mb: 512 + + # The AM is CPU-light, a single core will do. + + vcores: 1 + + # The AM uses no disk. Available on selected YARN distributions. + + disks: 0 + + # Heap memory for the AM. Estimate based on observation. + + heap: "450M" + + # Arguments passed to the AM JVM. (Private) + + vm-args: "" + + # AM-to-RM period in ms. (Private) + + poll-ms: 1000 + + # AM clock tick period in ms. (Private) + + tick-ms: 2000 + + # Set to true to dump launch variables and command to log. + + debug-launch: false + + # Extra class path for AM. The path must be valid on all nodes. + # Not normally used; provided only for special cases to avoid editing + # the launch script. This is a class path prefix to allow overrides + # of Drill classes. + # Analogous to DRILL_CLASSPATH_PREFIX in drill-config.sh + # Multiple entries should be separated by colons. + + prefix-class-path: "" + + # Extra class path for AM. The path must be valid on all nodes on which + # the AM may run. + # Not normally used; provided only for special cases to avoid editing + # the launch script. + + class-path: "" + + # Enables/disables auto shutdown when no Drillbits can run. Disable + # this when required for single-node testing. + + auto-shutdown: true + + # Specify nodes that can run the AM via a YARN node label expression. + # Blank means that node labele expression will be applied. + + node-label-expr: "" + } + + drillbit: { + + # Default memory: 4096 (heap) + 8192 (direct) + 1024 (code cache) + 1024 overhead. + + memory-mb: 14336 + vcores: 4 + disks: 1 + + heap: "4G" + max-direct-memory: "8G" + code-cache: "1G" + + # Additional JVM arguments for the drillbit (Private) + # The value here is appended to any set in the launch script. + # Separate multiple arguments by spaces. + + vm-args: "" + + # Set to true to turn on garbage collection logging. + # Similar to SERVER_GC_OPTS in drillbit.sh. However under YARN + # all logging goes to YARN's log directory and the GC log is + # always called "gc.log". + + log-gc: false + + # Extra class path for Drill-bit. The path must be valid on all nodes. + # Not normally used; provided only for special cases to avoid editing + # the launch script. This is a class path prefix to allow overrides + # of Drill classes. + # Equivalent to DRILL_CLASSPATH_PREFIX in drill-config.sh + # Multiple entries should be separated by colons. + + prefix-class-path: "" + + # Extra class path for Drill extensions such as Hadoop or HBase. + # The path must be valid on all nodes. Inserted into Drill's + # class path before Drill's own third-party extensions; allows + # overriding of Drill's own included jars. + # Equivalent to HADOOP_CLASSPATH and HBASE_CLASSPATH + # in drill-config.sh + + extn-class-path: "" + + # Normal class path specification for jars added to Drill's own. + # Appears in the class path after Drill's jars. Useful for dependencies + # on custom data source implementations. + # The path must be valid on all nodes. + # Equivalent to DRILL_CLASSPATH in drill-config.sh + + class-path: "" + + # Maximum number of retries of each Drillbit before blacklisting the node. + + max-retries: 3 + + # Set to true to dump launch variables and command to log. + + debug-launch: false + + # Disable normal YARN log location. Allows Drill to write to + # it's own log location as when running outside of YARN. However, + # under YARN, log aggregation is generally considered a "good thing." + + disable-yarn-logs: false + + # DoY maintains an approximate count of free nodes available under YARN. + # A common error is to ask for far more Drillbits than can be run given + # the available nodes. This setting limits the number of "extra" requests + # beyond the known free nodes. The reason this number is not zero is that + # the DoY count is approximate, so we want to provide so "wiggle room." + + max-extra-nodes: 2 + + # Maximum time that the AM will wait when requesting a container to run + # the Drillbit. After this time, the request will timeout and the AM will + # decrease the desired cluster size to match the fact that no additional + # node appears to be available. The timeout is in seconds. 0 means no + # timeout + + request-timeout-secs: 600 + } + + http: { + enabled: true + port: 8048 + ssl-enabled: false + auth-type: "none" + # Default of one hour, as in the Drillbit + session-max-idle-secs: 3600 + rest-key="secret" + docs-link: "http://drill.apache.org/docs/"; + refresh-secs: 5 + } + + cluster: [ + + # Defined to support a demo single-node cluster. + + { + name: "default" + type: "basic" + count: 1 + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestClient.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestClient.java b/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestClient.java new file mode 100644 index 0000000..7912bd5 --- /dev/null +++ b/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestClient.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.client; + +import static org.junit.Assert.*; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; + +import org.apache.drill.yarn.client.ClientContext; +import org.apache.drill.yarn.client.DrillOnYarn; +import org.junit.Test; + +public class TestClient { + + /** + * Unchecked exception to allow capturing "exit" events without actually + * exiting. + */ + + public static class SimulatedExitException extends RuntimeException { + private static final long serialVersionUID = 1L; + public int exitCode; + + public SimulatedExitException(int exitCode) { + this.exitCode = exitCode; + } + } + + public static class TestContext extends ClientContext { + public static ByteArrayOutputStream captureOut = new ByteArrayOutputStream(); + public static ByteArrayOutputStream captureErr = new ByteArrayOutputStream(); + + public static void testInit() { + init(new TestContext()); + resetOutput(); + } + + @Override + public void exit(int exitCode) { + throw new SimulatedExitException(exitCode); + } + + public static void resetOutput() { + try { + out.flush(); + captureOut.reset(); + out = new PrintStream(captureOut, true, "UTF-8"); + err.flush(); + captureErr.reset(); + err = new PrintStream(captureErr, true, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } + } + + public static String getOut() { + out.flush(); + try { + return captureOut.toString("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } + } + + public static String getErr() { + out.flush(); + try { + return captureErr.toString("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } + } + } + + /** + * Test the basics of the DrillOnYarn app. Does not try any real commands, but + * does check for the basic error conditions. + */ + + @Test + public void testBasics() { + TestContext.testInit(); + + // No arguments provided. + + try { + DrillOnYarn.run(new String[] {}); + fail(); + } catch (SimulatedExitException e) { + assert (e.exitCode == -1); + assertTrue(TestContext.getOut().contains("Usage: ")); + } + + // Bogus command + + try { + DrillOnYarn.run(new String[] { "bogus" }); + fail(); + } catch (SimulatedExitException e) { + assert (e.exitCode == -1); + assertTrue(TestContext.getOut().contains("Usage: ")); + } + + // Help command + + try { + DrillOnYarn.run(new String[] { "help" }); + fail(); + } catch (SimulatedExitException e) { + assert (e.exitCode == -1); + assertTrue(TestContext.getOut().contains("Usage: ")); + } + } + + // The idea here is to set up a simulated client environment, then + // test each command. This is a big project. + +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestCommandLineOptions.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestCommandLineOptions.java b/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestCommandLineOptions.java new file mode 100644 index 0000000..f996c5b --- /dev/null +++ b/drill-yarn/src/test/java/org/apache/drill/yarn/client/TestCommandLineOptions.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.client; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.apache.drill.yarn.client.CommandLineOptions; + +public class TestCommandLineOptions { + @Test + public void testOptions() { + CommandLineOptions opts = new CommandLineOptions(); + opts.parse(new String[] {}); + assertNull(opts.getCommand()); + + opts = new CommandLineOptions(); + opts.parse(new String[] { "-h" }); + assertEquals(CommandLineOptions.Command.HELP, opts.getCommand()); + + opts = new CommandLineOptions(); + opts.parse(new String[] { "-?" }); + assertEquals(CommandLineOptions.Command.HELP, opts.getCommand()); + + opts = new CommandLineOptions(); + opts.parse(new String[] { "help" }); + assertEquals(CommandLineOptions.Command.HELP, opts.getCommand()); + + opts = new CommandLineOptions(); + opts.parse(new String[] { "start" }); + assertEquals(CommandLineOptions.Command.START, opts.getCommand()); + + opts = new CommandLineOptions(); + opts.parse(new String[] { "stop" }); + assertEquals(CommandLineOptions.Command.STOP, opts.getCommand()); + + opts = new CommandLineOptions(); + opts.parse(new String[] { "status" }); + assertEquals(CommandLineOptions.Command.STATUS, opts.getCommand()); + + opts = new CommandLineOptions(); + opts.parse(new String[] { "resize" }); + assertNull(opts.getCommand()); + + opts = new CommandLineOptions(); + opts.parse(new String[] { "resize", "10" }); + assertEquals(CommandLineOptions.Command.RESIZE, opts.getCommand()); + assertEquals("", opts.getResizePrefix()); + assertEquals(10, opts.getResizeValue()); + + opts = new CommandLineOptions(); + opts.parse(new String[] { "resize", "+2" }); + assertEquals(CommandLineOptions.Command.RESIZE, opts.getCommand()); + assertEquals("+", opts.getResizePrefix()); + assertEquals(2, opts.getResizeValue()); + + opts = new CommandLineOptions(); + opts.parse(new String[] { "resize", "-3" }); + assertEquals(CommandLineOptions.Command.RESIZE, opts.getCommand()); + assertEquals("-", opts.getResizePrefix()); + assertEquals(3, opts.getResizeValue()); + + opts = new CommandLineOptions(); + opts.parse(new String[] { "myDrill" }); + assertNull(opts.getCommand()); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/core/TestConfig.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/core/TestConfig.java b/drill-yarn/src/test/java/org/apache/drill/yarn/core/TestConfig.java new file mode 100644 index 0000000..0519484 --- /dev/null +++ b/drill-yarn/src/test/java/org/apache/drill/yarn/core/TestConfig.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.core; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.junit.Test; + +import com.typesafe.config.Config; + +public class TestConfig { + + /** + * Mock config that lets us tinker with loading and environment access for + * testing. + */ + + private static class DoYTestConfig extends DrillOnYarnConfig { + protected Map<String, String> mockEnv = new HashMap<>(); + protected File configDir; + + public DoYTestConfig(TestClassLoader cl, File configDir) + throws DoyConfigException { + doLoad(cl); + instance = this; + this.configDir = configDir; + } + + @Override + protected String getEnv(String key) { + return mockEnv.get(key); + } + } + + /** + * Mock class loader to let us add config files after the JVM starts. (In + * production code, the config file directories are added to the class path.) + */ + + private static class TestClassLoader extends ClassLoader { + private File configDir; + + public TestClassLoader(ClassLoader parent, File configDir) { + super(parent); + this.configDir = configDir; + } + + @Override + protected URL findResource(String name) { + File file = new File(configDir, name); + if (file.exists()) { + try { + return file.toURI().toURL(); + } catch (MalformedURLException e) { + ; + } + } + return null; + } + } + + /** + * Creates a stack of settings to test overrides. + * <table> + * <tr> + * <th>property</th> + * <th>default</th> + * <th>distrib</th> + * <th>user</th> + * <th>system</th> + * </tr> + * <tr> + * <td>drill-key</td> + * <td>"drill"</td> + * <td></td> + * <td></td> + * <td></td> + * </tr> + * <tr> + * <td>app-dir</td> + * <td>"/user/drill"</td> + * <td>"/opt/drill"</td> + * <td></td> + * <td></td> + * </tr> + * <tr> + * <td>app-name</td> + * <td>"Drill-on-YARN"</td> + * <td>"config-app-name"</td> + * <td>"My-App"</td> + * <td></td> + * </tr> + * <tr> + * <td>queue</td> + * <td>"default"</td> + * <td>"distrib-queue"</td> + * <td>"my-queue"</td> + * <td>"sys-queue"</td> + * </table> + * <p> + * Full property names: + * <ul> + * <li>drill.yarn.drill-install.drill-key</li> + * <li>drill.yarn.dfs.app-dir</li> + * <li>drill.yarn.app-name</li> + * <li>drill.yarn.zk.connect</li> + * </ul> + * + * @throws IOException + * @throws DoyConfigException + */ + @Test + public void testLoad() throws IOException, DoyConfigException { + + DoYTestConfig doyConfig = initConfig("test-doy-config.conf"); + Config config = DrillOnYarnConfig.config(); + + assertEquals("drill", + config.getString(DrillOnYarnConfig.DRILL_ARCHIVE_KEY)); + assertEquals("/opt/drill", config.getString(DrillOnYarnConfig.DFS_APP_DIR)); + assertEquals("My-App", config.getString(DrillOnYarnConfig.APP_NAME)); + // Commenting out for now, fails on VM. + //assertEquals("sys-queue", config.getString(DrillOnYarnConfig.YARN_QUEUE)); + + // Should also have access to Drill options. + // Does not test Drill's override mechanism because have not found a good + // way to add drill-override.conf to the class path in this test. + + // assertEquals( "org.apache.drill.exec.opt.IdentityOptimizer", + // config.getString( "drill.exec.optimizer" ) ); + assertEquals("drillbits1", config.getString(DrillOnYarnConfig.CLUSTER_ID)); + + // Drill home: with and without an env var. + // Must set the site env var. Class path testing can't be done here. + // No DRILL_HOME: will only occur during testing. In that case, we use + // the setting from the config file. Explicit site dir. + + assertNull(doyConfig.mockEnv.get(DrillOnYarnConfig.DRILL_HOME_ENV_VAR)); + doyConfig.mockEnv.put(DrillOnYarnConfig.DRILL_SITE_ENV_VAR, "/drill/site"); + doyConfig.setClientPaths(); + assertEquals("/config/drill/home", + doyConfig.getLocalDrillHome().getAbsolutePath()); + assertTrue(doyConfig.hasSiteDir()); + assertEquals("/drill/site", doyConfig.getLocalSiteDir().getAbsolutePath()); + + // Home set in an env var + + doyConfig.mockEnv.put(DrillOnYarnConfig.DRILL_HOME_ENV_VAR, "/drill/home"); + doyConfig.setClientPaths(); + assertEquals("/drill/home", + doyConfig.getLocalDrillHome().getAbsolutePath()); + + // Remote site: localized case + + assertTrue(config.getBoolean(DrillOnYarnConfig.LOCALIZE_DRILL)); + assertEquals("/foo/bar/drill-archive.tar.gz", + config.getString(DrillOnYarnConfig.DRILL_ARCHIVE_PATH)); + assertEquals("$PWD/drill/drill-archive", doyConfig.getRemoteDrillHome()); + assertEquals("site", config.getString(DrillOnYarnConfig.SITE_ARCHIVE_KEY)); + assertEquals("$PWD/site", doyConfig.getRemoteSiteDir()); + + // Localized, but no separate site directory + + doyConfig.mockEnv.put(DrillOnYarnConfig.DRILL_SITE_ENV_VAR, + "/drill/home/conf"); + doyConfig.setClientPaths(); + // If $DRILL_HOME/conf is used, we still treat id as a site dir. +// assertFalse(doyConfig.hasSiteDir()); +// assertNull(doyConfig.getRemoteSiteDir()); + + // Local app id file: composed from Drill home, ZK root and cluster id. + // (Turns out that there can be two different clusters sharing the same + // root...) + // With no site dir, app id is in parent of the drill directory. + + assertEquals("/drill/home", + doyConfig.getLocalDrillHome().getAbsolutePath()); + assertEquals("drill", config.getString(DrillOnYarnConfig.ZK_ROOT)); + assertEquals("drillbits1", config.getString(DrillOnYarnConfig.CLUSTER_ID)); + assertEquals("/drill/home/drill-drillbits1.appid", + doyConfig.getLocalAppIdFile().getAbsolutePath()); + + // Again, but with a site directory. App id is in parent of the site + // directory. + + doyConfig.mockEnv.put(DrillOnYarnConfig.DRILL_SITE_ENV_VAR, + "/var/drill/site"); + doyConfig.setClientPaths(); + assertEquals("/var/drill/drill-drillbits1.appid", + doyConfig.getLocalAppIdFile().getAbsolutePath()); + } + + private DoYTestConfig initConfig(String configName) + throws IOException, DoyConfigException { + File tempDir = new File(System.getProperty("java.io.tmpdir")); + File configDir = new File(tempDir, "config"); + if (configDir.exists()) { + FileUtils.forceDelete(configDir); + } + configDir.mkdirs(); + configDir.deleteOnExit(); + + InputStream in = getClass().getResourceAsStream("/" + configName); + File dest = new File(configDir, "drill-on-yarn.conf"); + Files.copy(in, dest.toPath(), StandardCopyOption.REPLACE_EXISTING); + in = getClass().getResourceAsStream("/test-doy-distrib.conf"); + dest = new File(configDir, "doy-distrib.conf"); + Files.copy(in, dest.toPath(), StandardCopyOption.REPLACE_EXISTING); + + System.setProperty(DrillOnYarnConfig.YARN_QUEUE, "sys-queue"); + + TestClassLoader cl = new TestClassLoader(this.getClass().getClassLoader(), + configDir); + + assertNotNull(cl.getResource(DrillOnYarnConfig.DISTRIB_FILE_NAME)); + + return new DoYTestConfig(cl, configDir); + } + + @Test + public void testNonLocalized() throws IOException, DoyConfigException { + DoYTestConfig doyConfig = initConfig("second-test-config.conf"); + + // Test the non-localized case + + doyConfig.mockEnv.put(DrillOnYarnConfig.DRILL_SITE_ENV_VAR, "/drill/site"); + doyConfig.setClientPaths(); + assertEquals("/config/drill/home", doyConfig.getRemoteDrillHome()); + assertEquals("/config/drill/site", doyConfig.getRemoteSiteDir()); + } + + @Test + public void testNonLocalizedNonSite() throws IOException, DoyConfigException { + DoYTestConfig doyConfig = initConfig("third-test-config.conf"); + + // Test the non-localized case + + assertEquals("/config/drill/home", doyConfig.getRemoteDrillHome()); + assertNull(doyConfig.getRemoteSiteDir()); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/README.md ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/README.md b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/README.md new file mode 100644 index 0000000..3e4017b --- /dev/null +++ b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/README.md @@ -0,0 +1,65 @@ +# Script Test Overview + +The tests here exercise the Drill shell scripts with a wide variety of options. +The Drill scripts are designed to be run from the command line or from YARN. +The scripts allow passing in values from a vendor-specific configuration file +($DRILL_HOME/conf/distrib-env.sh), a user-specific configuration file +($DRILL_SITE/drill-env.sh) or from environment variables set by YARN. + +Testing scripts is normally tedious because the scripts are designed to start +a process, perhaps a Drillbit or Sqlline. To make automated tests possible, +the scripts incorporate a "shim": an environment variable that, if set, is +used to put a "wrapper" script around the Java execution line. The wrapper +captures the environment and the command line, and generates stderr and +stdout output. The test progams use this captured output to determine if +the Java command line has the options we expect. (We boldly assume that +if we give Java the right options, it will do the right thing with them.) + +Why are the script tests in the drill-yarn project? Because YARN is the most +sensitive to any changes: YARN provides several levels of indirection between +the user and the scripts; the scripts must work exactly as the Drill-on-YARN +code expects (or visa-versa) or things tend to break. + +## Inputs + +The test program needs the following inputs: + +- The scripts, which are copied from the source tree. (Need details) +- /src/test/resources/wrapper.sh which is the "magic" wrapper script +for capturing the command line, etc. +- A temporary directory where the program can build its mock Drill +and site directories. + +## Running the Tests + +Simply run the tests. Each test sets up its distribution and +optional site directory and required environment variables. Each +test uses a builder to build up the required script launch and +to analyze the results. Each test function usually does a single +setup, then does a bunch of test runs against that environment. + +Each test uses "gobbler" threads to read the stdout and stderr +from the test run. If you run the test in a debugger, you'll +see a steady stream of threads come and go. This is a test, so +we don't bother with a thread pool; we just brute-force create +new threads as needed. + +## Extending the Tests + +You should extend the tests if you: + +- Add new environment variables to any script. +- Change the logic of any script. +- Find a bug that these tests somehow did not catch. + +Note that it is very important to ensure that each new enviornment +variable or other option works when set in distrib-env.sh, +drill-env.sh or in the environment. These tests are the only (sane) +way to test the many combinations, and to do that on each +subsequent change. + +## About the Code + +TestScripts.java are the tests, organized by functional area. ScriptUtils.java +is a large number of (mostly ad-hoc) utilities needed to set up, run, tear down +and analyze the results of each run. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/f2ac8749/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/ScriptUtils.java ---------------------------------------------------------------------- diff --git a/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/ScriptUtils.java b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/ScriptUtils.java new file mode 100644 index 0000000..3517cf8 --- /dev/null +++ b/drill-yarn/src/test/java/org/apache/drill/yarn/scripts/ScriptUtils.java @@ -0,0 +1,847 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.yarn.scripts; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import org.apache.commons.io.FileUtils; + +public class ScriptUtils { + + private static ScriptUtils instance = new ScriptUtils(); + public File distribDir; + public File javaHome; + public File testDir; + public File testDrillHome; + public File testSiteDir; + public File testLogDir; + public boolean externalLogDir; + + /** + * Out-of-the-box command-line arguments when launching sqlline. + * Order is not important here (though it is to Java.) + */ + + public static String sqlLineArgs[] = makeSqlLineArgs( ); + + private static String[] makeSqlLineArgs( ) { + String args[] = { + "-Dlog.path=/.*/drill/log/sqlline\\.log", + "-Dlog.query.path=/.*/drill/log/sqlline_queries\\.json", + "-XX:MaxPermSize=512M", + "sqlline\\.SqlLine", + "-d", + "org\\.apache\\.drill\\.jdbc\\.Driver", + "--maxWidth=10000", + "--color=true" + }; + + // Special handling if this machine happens to have the default + // /var/log/drill log location. + + if ( new File( "/var/log/drill" ).exists() ) { + args[ 0 ] = "-Dlog\\.path=/var/log/drill/sqlline\\.log"; + args[ 1 ] = "-Dlog\\.query\\.path=/var/log/drill/sqlline_queries\\.json"; + } + return args; + } + + public static final boolean USE_SOURCE = true; + public static final String TEMP_DIR = "/tmp"; + public static boolean useSource = USE_SOURCE; + + private ScriptUtils() { + String drillScriptsDir = System.getProperty("drillScriptDir"); + assertNotNull(drillScriptsDir); + distribDir = new File(drillScriptsDir); + javaHome = new File(System.getProperty("java.home")); + } + + public static ScriptUtils instance() { + return instance; + } + + public ScriptUtils fromSource(String sourceDir) { + useSource = true; + return this; + } + + public ScriptUtils fromDistrib(String distrib) { + distribDir = new File(distrib); + useSource = false; + return this; + } + + /** + * Out-of-the-box command-line arguments when launching Drill. + * Order is not important here (though it is to Java.) + */ + + public static String stdArgs[] = buildStdArgs( ); + + private static String[] buildStdArgs( ) + { + String args[] = { + "-Xms4G", + "-Xmx4G", + "-XX:MaxDirectMemorySize=8G", + "-XX:MaxPermSize=512M", + "-XX:ReservedCodeCacheSize=1G", + // Removed in Drill 1.8 +// "-Ddrill\\.exec\\.enable-epoll=true", + "-XX:\\+CMSClassUnloadingEnabled", + "-XX:\\+UseG1GC", + "org\\.apache\\.drill\\.exec\\.server\\.Drillbit", + "-Dlog\\.path=/.*/script-test/drill/log/drillbit\\.log", + "-Dlog\\.query\\.path=/.*/script-test/drill/log/drillbit_queries\\.json", + }; + + // Special handling if this machine happens to have the default + // /var/log/drill log location. + + if ( new File( "/var/log/drill" ).exists() ) { + args[ args.length-2 ] = "-Dlog\\.path=/var/log/drill/drillbit\\.log"; + args[ args.length-1 ] = "-Dlog\\.query\\.path=/var/log/drill/drillbit_queries\\.json"; + } + return args; + }; + + /** + * Out-of-the-box class-path before any custom additions. + */ + + static String stdCp[] = + { + "conf", + "jars/*", + "jars/ext/*", + "jars/3rdparty/*", + "jars/classb/*" + }; + + /** + * Directories to create to simulate a Drill distribution. + */ + + static String distribDirs[] = { + "bin", + "jars", + "jars/3rdparty", + "jars/ext", + "conf" + }; + + /** + * Out-of-the-box Jar directories. + */ + + static String jarDirs[] = { + "jars", + "jars/3rdparty", + "jars/ext", + }; + + /** + * Scripts we must copy from the source tree to create a simulated + * Drill bin directory. + */ + + public static String scripts[] = { + "drill-config.sh", + "drill-embedded", + "drill-localhost", + "drill-on-yarn.sh", + "drillbit.sh", + "drill-conf", + //dumpcat + //hadoop-excludes.txt + "runbit", + "sqlline", + //sqlline.bat + //submit_plan + "yarn-drillbit.sh" + }; + + /** + * Create the basic test directory. Tests add or remove details. + */ + + public void initialSetup() throws IOException { + File tempDir = new File(TEMP_DIR); + testDir = new File(tempDir, "script-test"); + testDrillHome = new File(testDir, "drill"); + testSiteDir = new File(testDir, "site"); + File varLogDrill = new File( "/var/log/drill" ); + if ( varLogDrill.exists() ) { + testLogDir = varLogDrill; + externalLogDir = true; + } else { + testLogDir = new File(testDrillHome, "log"); + } + if (testDir.exists()) { + FileUtils.forceDelete(testDir); + } + testDir.mkdirs(); + testSiteDir.mkdir(); + testLogDir.mkdir(); + } + + public void createMockDistrib() throws IOException { + if (ScriptUtils.useSource) { + buildFromSource(); + } else { + buildFromDistrib(); + } + } + + /** + * Build the Drill distribution directory directly from sources. + */ + + private void buildFromSource() throws IOException { + createMockDirs(); + copyScripts(ScriptUtils.instance().distribDir); + } + + /** + * Build the shell of a Drill distribution directory by creating the required + * directory structure. + */ + + private void createMockDirs() throws IOException { + if (testDrillHome.exists()) { + FileUtils.forceDelete(testDrillHome); + } + testDrillHome.mkdir(); + for (String path : ScriptUtils.distribDirs) { + File subDir = new File(testDrillHome, path); + subDir.mkdirs(); + } + for (String path : ScriptUtils.jarDirs) { + makeDummyJar(new File(testDrillHome, path), "dist"); + } + } + + /** + * The tests should not require jar files, but we simulate them to be a bit + * more realistic. Since we dont' run Java, the jar files can be simulated. + */ + + public File makeDummyJar(File dir, String prefix) throws IOException { + String jarName = ""; + if (prefix != null) { + jarName += prefix + "-"; + } + jarName += dir.getName() + ".jar"; + File jarFile = new File(dir, jarName); + writeFile(jarFile, "Dummy jar"); + return jarFile; + } + + /** + * Create a simple text file with the given contents. + */ + + public void writeFile(File file, String contents) throws IOException { + try (PrintWriter out = new PrintWriter(new FileWriter(file))) { + out.println(contents); + } + } + + /** + * Create a drill-env.sh or distrib-env.sh file with the given environment in + * the recommended format. + */ + + public void createEnvFile(File file, Map<String, String> env) + throws IOException { + try (PrintWriter out = new PrintWriter(new FileWriter(file))) { + out.println("#!/usr/bin/env bash"); + for (String key : env.keySet()) { + String value = env.get(key); + out.print("export "); + out.print(key); + out.print("=${"); + out.print(key); + out.print(":-\""); + out.print(value); + out.println("\"}"); + } + } + } + + /** + * Copy the standard scripts from source location to the mock distribution + * directory. + */ + + private void copyScripts(File sourceDir) throws IOException { + File binDir = new File(testDrillHome, "bin"); + for (String script : ScriptUtils.scripts) { + File source = new File(sourceDir, script); + File dest = new File(binDir, script); + copyFile(source, dest); + dest.setExecutable(true); + } + + // Create the "magic" wrapper script that simulates the Drillbit and + // captures the output we need for testing. + + String wrapper = "wrapper.sh"; + File dest = new File(binDir, wrapper); + try (InputStream is = getClass().getResourceAsStream("/" + wrapper)) { + Files.copy(is, dest.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + dest.setExecutable(true); + } + + private void buildFromDistrib() { + // TODO Auto-generated method stub + + } + + /** + * Consume the input from a stream, specifically the stderr or stdout stream + * from a process. + * + * @see http://stackoverflow.com/questions/14165517/processbuilder-forwarding-stdout-and-stderr-of-started-processes-without-blocki + */ + + private static class StreamGobbler extends Thread { + InputStream is; + public StringBuilder buf = new StringBuilder(); + + private StreamGobbler(InputStream is) { + this.is = is; + } + + @Override + public void run() { + try { + InputStreamReader isr = new InputStreamReader(is); + BufferedReader br = new BufferedReader(isr); + String line = null; + while ((line = br.readLine()) != null) { + buf.append(line); + buf.append("\n"); + } + } catch (IOException ioe) { + ioe.printStackTrace(); + } + } + } + + /** + * Handy run result class to capture the information we need for testing and + * to do various kinds of validation on it. + */ + + public static class RunResult { + File logDir; + File logFile; + String stdout; + String stderr; + List<String> echoArgs; + int returnCode; + String classPath[]; + String libPath[]; + String log; + public File pidFile; + public File outFile; + String out; + + /** + * Split the class path into strings for easier validation. + */ + + public void analyze() { + if (echoArgs == null) { + return; + } + for (int i = 0; i < echoArgs.size(); i++) { + String arg = echoArgs.get(i); + if (arg.equals("-cp")) { + classPath = Pattern.compile(":").split((echoArgs.get(i + 1))); + break; + } + } + String probe = "-Djava.library.path="; + for (int i = 0; i < echoArgs.size(); i++) { + String arg = echoArgs.get(i); + if (arg.startsWith(probe)) { + assertNull(libPath); + libPath = Pattern.compile(":").split((arg.substring(probe.length()))); + break; + } + } + } + + /** + * Read the log file, if any, generated by the process. + */ + + public void loadLog() throws IOException { + log = loadFile(logFile); + } + + private String loadFile(File file) throws IOException { + StringBuilder buf = new StringBuilder(); + try ( BufferedReader reader = new BufferedReader(new FileReader(file)) ) { + String line; + while ((line = reader.readLine()) != null) { + buf.append(line); + buf.append("\n"); + } + return buf.toString(); + } catch (FileNotFoundException e) { + return null; + } + } + + /** + * Validate that the first argument invokes Java correctly. + */ + + public void validateJava() { + assertNotNull(echoArgs); + String java = instance.javaHome + "/bin/java"; + List<String> actual = echoArgs; + assertEquals(java, actual.get(0)); + } + + public boolean containsArg(String arg) { + for (String actual : echoArgs) { + if (actual.equals(arg)) { + return true; + } + } + return false; + } + + public void validateStockArgs() { + for (String arg : ScriptUtils.stdArgs) { + assertTrue("Argument not found: " + arg + " in " + echoArgs, containsArgRegex(arg)); + } + } + + public void validateArg(String arg) { + validateArgs(Collections.singletonList(arg)); + } + + public void validateArgs(String args[]) { + validateArgs(Arrays.asList(args)); + } + + public void validateArgs(List<String> args) { + validateJava(); + for (String arg : args) { + assertTrue(containsArg(arg)); + } + } + + public void validateArgRegex(String arg) { + assertTrue(containsArgRegex(arg)); + } + + public void validateArgsRegex(List<String> args) { + assertTrue(containsArgsRegex(args)); + } + + public boolean containsArgsRegex(List<String> args) { + for (String arg : args) { + if (!containsArgRegex(arg)) { + return false; + } + } + return true; + } + + public boolean containsArgsRegex(String args[]) { + for (String arg : args) { + if (!containsArgRegex(arg)) { + return false; + } + } + return true; + } + + public boolean containsArgRegex(String arg) { + for (String actual : echoArgs) { + if (actual.matches(arg)) { + return true; + } + } + return false; + } + + public void validateClassPath(String expectedCP) { + assertTrue(classPathContains(expectedCP)); + } + + public void validateClassPath(String expectedCP[]) { + assertTrue(classPathContains(expectedCP)); + } + + public boolean classPathContains(String expectedCP[]) { + for (String entry : expectedCP) { + if (!classPathContains(entry)) { + return false; + } + } + return true; + } + + public boolean classPathContains(String expectedCP) { + if (classPath == null) { + fail("No classpath returned"); + } + String tail = "/" + instance.testDir.getName() + "/" + + instance.testDrillHome.getName() + "/"; + String expectedPath; + if (expectedCP.startsWith("/")) { + expectedPath = expectedCP; + } else { + expectedPath = tail + expectedCP; + } + for (String entry : classPath) { + if (entry.endsWith(expectedPath)) { + return true; + } + } + return false; + } + + public void loadOut() throws IOException { + out = loadFile(outFile); + } + + /** + * Ensure that the Drill log file contains at least the sample message + * written by the wrapper. + */ + + public void validateDrillLog() { + assertNotNull(log); + assertTrue(log.contains("Drill Log Message")); + } + + /** + * Validate that the stdout contained the expected message. + */ + + public void validateStdOut() { + assertTrue(stdout.contains("Starting drillbit on")); + } + + /** + * Validate that the stderr contained the sample error message from the + * wrapper. + */ + + public void validateStdErr() { + assertTrue(stderr.contains("Stderr Message")); + } + + public int getPid() throws IOException { + try (BufferedReader reader = new BufferedReader(new FileReader(pidFile))) { + return Integer.parseInt(reader.readLine()); + } + finally { } + } + + } + + /** + * The "business end" of the tests: runs drillbit.sh and captures results. + */ + + public static class ScriptRunner { + // Drillbit commands + + public static String DRILLBIT_RUN = "run"; + public static String DRILLBIT_START = "start"; + public static String DRILLBIT_STATUS = "status"; + public static String DRILLBIT_STOP = "stop"; + public static String DRILLBIT_RESTART = "restart"; + + public File cwd = instance.testDir; + public File drillHome = instance.testDrillHome; + public String script; + public List<String> args = new ArrayList<>(); + public Map<String, String> env = new HashMap<>(); + public File logDir; + public File pidFile; + public File outputFile; + public boolean preserveLogs; + + public ScriptRunner(String script) { + this.script = script; + } + + public ScriptRunner(String script, String cmd) { + this(script); + args.add(cmd); + } + + public ScriptRunner(String script, String cmdArgs[]) { + this(script); + for (String arg : cmdArgs) { + args.add(arg); + } + } + + public ScriptRunner withArg(String arg) { + args.add(arg); + return this; + } + + public ScriptRunner withSite(File siteDir) { + if (siteDir != null) { + args.add("--site"); + args.add(siteDir.getAbsolutePath()); + } + return this; + } + + public ScriptRunner withEnvironment(Map<String, String> env) { + if (env != null) { + this.env.putAll(env); + } + return this; + } + + public ScriptRunner addEnv(String key, String value) { + env.put(key, value); + return this; + } + + public ScriptRunner withLogDir(File logDir) { + this.logDir = logDir; + return this; + } + + public ScriptRunner preserveLogs() { + preserveLogs = true; + return this; + } + + public RunResult run() throws IOException { + File binDir = new File(drillHome, "bin"); + File scriptFile = new File(binDir, script); + assertTrue(scriptFile.exists()); + outputFile = new File(instance.testDir, "output.txt"); + outputFile.delete(); + if (logDir == null) { + logDir = instance.testLogDir; + } + if (!preserveLogs) { + cleanLogs(logDir); + } + + Process proc = startProcess(scriptFile); + RunResult result = runProcess(proc); + if (result.returnCode == 0) { + captureOutput(result); + captureLog(result); + } + return result; + } + + private void cleanLogs(File logDir) throws IOException { + if ( logDir == instance.testLogDir && instance.externalLogDir ) { + return; + } + if (logDir.exists()) { + FileUtils.forceDelete(logDir); + } + } + + private Process startProcess(File scriptFile) throws IOException { + outputFile.delete(); + List<String> cmd = new ArrayList<>(); + cmd.add(scriptFile.getAbsolutePath()); + cmd.addAll(args); + ProcessBuilder pb = new ProcessBuilder().command(cmd).directory(cwd); + Map<String, String> pbEnv = pb.environment(); + pbEnv.clear(); + pbEnv.putAll(env); + File binDir = new File(drillHome, "bin"); + File wrapperCmd = new File(binDir, "wrapper.sh"); + + // Set the magic wrapper to capture output. + + pbEnv.put("_DRILL_WRAPPER_", wrapperCmd.getAbsolutePath()); + pbEnv.put("JAVA_HOME", instance.javaHome.getAbsolutePath()); + return pb.start(); + } + + private RunResult runProcess(Process proc) { + StreamGobbler errorGobbler = new StreamGobbler(proc.getErrorStream()); + StreamGobbler outputGobbler = new StreamGobbler(proc.getInputStream()); + outputGobbler.start(); + errorGobbler.start(); + + try { + proc.waitFor(); + } catch (InterruptedException e) { + // Won't occur. + } + + RunResult result = new RunResult(); + result.stderr = errorGobbler.buf.toString(); + result.stdout = outputGobbler.buf.toString(); + result.returnCode = proc.exitValue(); + return result; + } + + private void captureOutput(RunResult result) throws IOException { + // Capture the Java arguments which the wrapper script wrote to a file. + + try (BufferedReader reader = new BufferedReader(new FileReader(outputFile))) { + result.echoArgs = new ArrayList<>(); + String line; + while ((line = reader.readLine()) != null) { + result.echoArgs.add(line); + } + result.analyze(); + } catch (FileNotFoundException e) { + ; + } + } + + private void captureLog(RunResult result) throws IOException { + result.logDir = logDir; + result.logFile = new File(logDir, "drillbit.log"); + if (result.logFile.exists()) { + result.loadLog(); + } else { + result.logFile = null; + } + } + } + + public static class DrillbitRun extends ScriptRunner { + public File pidDir; + + public DrillbitRun() { + super("drillbit.sh"); + } + + public DrillbitRun(String cmd) { + super("drillbit.sh", cmd); + } + + public DrillbitRun withPidDir(File pidDir) { + this.pidDir = pidDir; + return this; + } + + public DrillbitRun asDaemon() { + addEnv("KEEP_RUNNING", "1"); + return this; + } + + public RunResult start() throws IOException { + if (pidDir == null) { + pidDir = drillHome; + } + pidFile = new File(pidDir, "drillbit.pid"); + // pidFile.delete(); + asDaemon(); + RunResult result = run(); + if (result.returnCode == 0) { + capturePidFile(result); + captureDrillOut(result); + } + return result; + } + + private void capturePidFile(RunResult result) { + assertTrue(pidFile.exists()); + result.pidFile = pidFile; + } + + private void captureDrillOut(RunResult result) throws IOException { + // Drillbit.out + + result.outFile = new File(result.logDir, "drillbit.out"); + if (result.outFile.exists()) { + result.loadOut(); + } else { + result.outFile = null; + } + } + + } + + /** + * Build a "starter" conf or site directory by creating a mock + * drill-override.conf file. + */ + + public void createMockConf(File siteDir) throws IOException { + createDir(siteDir); + File override = new File(siteDir, "drill-override.conf"); + writeFile(override, "# Dummy override"); + } + + public void removeDir(File dir) throws IOException { + if (dir.exists()) { + FileUtils.forceDelete(dir); + } + } + + /** + * Remove, then create a directory. + */ + + public File createDir(File dir) throws IOException { + removeDir(dir); + dir.mkdirs(); + return dir; + } + + public void copyFile(File source, File dest) throws IOException { + Files.copy(source.toPath(), dest.toPath(), + StandardCopyOption.REPLACE_EXISTING); + } + +}
