Added: incubator/brooklyn/site/v/latest/ops/troubleshooting/going-deep-in-java-and-logs.html URL: http://svn.apache.org/viewvc/incubator/brooklyn/site/v/latest/ops/troubleshooting/going-deep-in-java-and-logs.html?rev=1693583&view=auto ============================================================================== --- incubator/brooklyn/site/v/latest/ops/troubleshooting/going-deep-in-java-and-logs.html (added) +++ incubator/brooklyn/site/v/latest/ops/troubleshooting/going-deep-in-java-and-logs.html Fri Jul 31 12:23:59 2015 @@ -0,0 +1,1151 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd";> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en"> + + +<head> + + +<title>Troubleshooting: Going Deep in Java and Logs - Apache Brooklyn (incubating)</title> + +<meta http-equiv="content-type" content="text/html; charset=utf-8"> +<meta name="viewport" content="width=device-width, initial-scale=1"> + +<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css"; rel="stylesheet"> +<link href="/v/latest/style/deps/octicons/octicons.css" rel="stylesheet"> +<link href="/v/latest/style/deps/bootstrap-theme.css" rel="stylesheet"> + +<link rel="stylesheet" href="/v/latest/style/css/code.css" type="text/css" media="screen" /> + +<link href="/v/latest/style/css/website.css" rel="stylesheet"> + +<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js";></script> +<script src="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js";></script> +<script type="text/javascript" src="/v/latest/style/deps/jquery.cookie.js"></script> + + +</head> + + +<body> + +<nav class="navbar navbar-default navbar-fixed-top" id="header" role="navigation"> + <div class="container-and-sidebars"> + + <div class="container-sidebar-left feather"> + <a href="http://www.apache.org/";> + <img src="/v/latest/style/img/feather.png" alt="[Apache]" width="80" class="flip navbar-feather"> + </a> + </div> + + <div class="container container-between-sidebars top-menu"> + <div class="container-fluid"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="/"><img src="/v/latest/style/img/apache-brooklyn-logo-244px-wide.png" alt="brooklyn"></a> + </div> + + <!-- Collect the nav links, forms, and other content for toggling --> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav navbar-right"> + + + <li class="dropdown"> + <a href="/learnmore/index.html">learn more</a> + <ul class="dropdown-menu" role="menu"> + <li> + <a href="/learnmore/index.html">Learn More</a> + </li> + <li class="divider"></li> + + <li> + + <a href="/learnmore/blueprint-tour.html">Blueprint Tour + </a> + + </li> + + <li> + + <a href="/learnmore/features/index.html">Features + </a> + + </li> + + <li> + + <a href="/learnmore/theory.html">Theory + </a> + + </li> + + <li> + + <a href="/learnmore/catalog/index.html">Browse Catalog + </a> + + </li> + + </ul> + </li> + + + + <li class=""> + <a href="/download/index.html">download</a> + </li> + + + + <li class=""> + <a href="/v/latest/start/running.html">get started</a> + </li> + + + + <li class="dropdown active"> + <a href="/documentation/index.html">documentation</a> + <ul class="dropdown-menu" role="menu"> + <li> + <a href="/documentation/index.html">Documentation</a> + </li> + <li class="divider"></li> + + <li> + + <a href="/v/latest/index.html">User Guide + </a> + + <div class="dropdown_section_header"><hr></div> + + </li> + + <li> + + <a href="/v/latest/yaml/creating-yaml.html">YAML Blueprints + </a> + + </li> + + <li> + + <a href="/v/latest/java/index.html">Java Blueprints + </a> + + </li> + + <li> + + <a href="/v/latest/ops/index.html">Operations + </a> + + <div class="dropdown_section_header"><hr></div> + + </li> + + <li> + + <a href="/v/latest/dev/index.html">Developer Guide + </a> + + </li> + + <li> + + <div class="dropdown_new_section"><hr></div> + + <a href="/documentation/other-docs.html">Other Resources + </a> + + </li> + + </ul> + </li> + + + + <li class="dropdown"> + <a href="/community/index.html">community</a> + <ul class="dropdown-menu" role="menu"> + <li> + <a href="/community/index.html">Community</a> + </li> + <li class="divider"></li> + + <li> + + <a href="/community/mailing-lists.html">Mailing Lists + </a> + + </li> + + <li> + + <a href="/community/irc.html">IRC + </a> + + </li> + + <li> + + <a href="https://issues.apache.org/jira/browse/BROOKLYN";>Bug Tracker (JIRA) + <span class="octicon octicon-link-external"></span></a> + + </li> + + <li> + + <a href="/community/how-to-contribute-docs.html">How to Contribute Documentation + </a> + + </li> + + </ul> + </li> + + + + <li class="dropdown"> + <a href="/developers/index.html">developers</a> + <ul class="dropdown-menu" role="menu"> + <li> + <a href="/developers/index.html">Developers</a> + </li> + <li class="divider"></li> + + <li> + + <a href="/developers/how-to-contribute.html">How to Contribute + </a> + + </li> + + <li> + + <a href="/v/latest/dev/index.html">Developer Guide + </a> + + </li> + + <li> + + <a href="/developers/committers/index.html">Committer Guide + </a> + + </li> + + <li> + + <a href="/developers/code-standards.html">Code Standards + </a> + + </li> + + <li> + + <a href="http://github.com/apache/incubator-brooklyn";>GitHub + <span class="octicon octicon-link-external"></span></a> + + </li> + + <li> + + <a href="https://issues.apache.org/jira/browse/BROOKLYN";>Bug Tracker (JIRA) + <span class="octicon octicon-link-external"></span></a> + + </li> + + </ul> + </li> + + + </ul> + </div><!-- /.navbar-collapse --> + </div><!-- /.container-fluid --> + </div><!-- /.container --> + + <div class="container-sidebar-right"> + <div class="navbar-sidebar-right-icons"> + <a href="https://github.com/apache/incubator-brooklyn"; class="navbar-icon navbar-icon-shift icon-github" + data-toggle="tooltip" data-placement="bottom" title="GitHub: apache/incubator-brooklyn"/> + <a href="https://twitter.com/#!/search?q=brooklyncentral"; class="navbar-icon navbar-icon-shift icon-twitter" + data-toggle="tooltip" data-placement="bottom" title="Twitter: @brooklyncentral"/> + <a href="http://webchat.freenode.net/?channels=brooklyncentral"; class="navbar-icon icon-irc" + data-toggle="tooltip" data-placement="bottom" title="IRC: freenode #brooklyncentral"/> + <!-- extra a element seems needed as landing page seems to copy the last element here (!?) + --> + <a href="/" style="width: 0px; height: 0px;"></a> + </div> + </div> + + </div> +</nav> + + +<div class="container" id="main_container"> + <div class="row"> + <div class="col-md-9" id="content_container"> + <div id="page_notes"></div> + <h1>Troubleshooting: Going Deep in Java and Logs</h1> + <p>This guide takes a deep look at the Java and log messages for some failure scenarios, +giving common steps used to identify the issues.</p> + +<h2 id="script-failure">Script Failure</h2> + +<p>Many blueprints run bash scripts as part of the installation. This section highlights how to identify a problem with +a bash script.</p> + +<p>First letâs take a look at the <code>customize()</code> method of the Tomcat server blueprint:</p> + +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="nd">@Override</span> + <span class="kd">public</span> <span class="kt">void</span> <span class="nf">customize</span><span class="o">()</span> <span class="o">{</span> + <span class="n">newScript</span><span class="o">(</span><span class="n">CUSTOMIZING</span><span class="o">)</span> + <span class="o">.</span><span class="na">body</span><span class="o">.</span><span class="na">append</span><span class="o">(</span><span class="s">"mkdir -p conf logs webapps temp"</span><span class="o">)</span> + <span class="o">.</span><span class="na">failOnNonZeroResultCode</span><span class="o">()</span> + <span class="o">.</span><span class="na">execute</span><span class="o">();</span> + + <span class="n">copyTemplate</span><span class="o">(</span><span class="n">entity</span><span class="o">.</span><span class="na">getConfig</span><span class="o">(</span><span class="n">TomcatServer</span><span class="o">.</span><span class="na">SERVER_XML_RESOURCE</span><span class="o">),</span> <span class="n">Os</span><span class="o">.</span><span class="na">mergePaths</span><span class="o">(</span><span class="n">getRunDir</span><span class="o">(),</span> <span class="s">"conf"</span><span class="o">,</span> <span class="s">"server.xml"</span><span class="o">));</span> + <span class="n">copyTemplate</span><span class="o">(</span><span class="n">entity</span><span class="o">.</span><span class="na">getConfig</span><span class="o">(</span><span class="n">TomcatServer</span><span class="o">.</span><span class="na">WEB_XML_RESOURCE</span><span class="o">),</span> <span class="n">Os</span><span class="o">.</span><span class="na">mergePaths</span><span class="o">(</span><span class="n">getRunDir</span><span class="o">(),</span> <span class="s">"conf"</span><span class="o">,</span> <span class="s">"web.xml"</span><span class="o">));</span> + + <span class="k">if</span> <span class="o">(</span><span class="n">isProtocolEnabled</span><span class="o">(</span><span class="s">"HTTPS"</span><span class="o">))</span> <span class="o">{</span> + <span class="n">String</span> <span class="n">keystoreUrl</span> <span class="o">=</span> <span class="n">Preconditions</span><span class="o">.</span><span class="na">checkNotNull</span><span class="o">(</span><span class="n">getSslKeystoreUrl</span><span class="o">(),</span> <span class="s">"keystore URL must be specified if using HTTPS for "</span> <span class="o">+</span> <span class="n">entity</span><span class="o">);</span> + <span class="n">String</span> <span class="n">destinationSslKeystoreFile</span> <span class="o">=</span> <span class="n">getHttpsSslKeystoreFile</span><span class="o">();</span> + <span class="n">InputStream</span> <span class="n">keystoreStream</span> <span class="o">=</span> <span class="n">resource</span><span class="o">.</span><span class="na">getResourceFromUrl</span><span class="o">(</span><span class="n">keystoreUrl</span><span class="o">);</span> + <span class="n">getMachine</span><span class="o">().</span><span class="na">copyTo</span><span class="o">(</span><span class="n">keystoreStream</span><span class="o">,</span> <span class="n">destinationSslKeystoreFile</span><span class="o">);</span> + <span class="o">}</span> + + <span class="n">getEntity</span><span class="o">().</span><span class="na">deployInitialWars</span><span class="o">();</span> + <span class="o">}</span></code></pre></div> + +<p>Here we can see that itâs running a script to create four directories before continuing with the customization. Letâs +introduce an error by changing <code>mkdir</code> to <code>mkrid</code>:</p> + +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">newScript</span><span class="o">(</span><span class="n">CUSTOMIZING</span><span class="o">)</span> + <span class="o">.</span><span class="na">body</span><span class="o">.</span><span class="na">append</span><span class="o">(</span><span class="s">"mkrid -p conf logs webapps temp"</span><span class="o">)</span> <span class="c1">// `mkdir` changed to `mkrid`</span> + <span class="o">.</span><span class="na">failOnNonZeroResultCode</span><span class="o">()</span> + <span class="o">.</span><span class="na">execute</span><span class="o">();</span></code></pre></div> + +<p>Now letâs try deploying this using the following YAML:</p> + +<div class="highlight"><pre><code class="language-yaml" data-lang="yaml"><span class="l-Scalar-Plain">name</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">Tomcat failure test</span> +<span class="l-Scalar-Plain">location</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">localhost</span> +<span class="l-Scalar-Plain">services</span><span class="p-Indicator">:</span> +<span class="p-Indicator">-</span> <span class="l-Scalar-Plain">type</span><span class="p-Indicator">:</span> <span class="l-Scalar-Plain">brooklyn.entity.webapp.tomcat.TomcatServer</span></code></pre></div> + +<p>Shortly after deployment, the entity fails with the following error:</p> + +<p><code>Failure running task ssh: customizing TomcatServerImpl{id=e1HP2s8x} (HmyPAozV): +Execution failed, invalid result 127 for customizing TomcatServerImpl{id=e1HP2s8x}</code></p> + +<p><a href="images/script-failure-large.png"><img src="images/script-failure.png" alt="Script failure error in the Brooklyn debug console." /></a></p> + +<p>By selecting the <code>Activities</code> tab, we can drill into the task that failed. The list of tasks shown (where the +effectors are shown as top-level tasks) are clickable links. Selecting that row will show the details of +that particular task, including its sub-tasks. We can eventually get to the specific sub-task that failed:</p> + +<p><a href="images/failed-task-large.png"><img src="images/failed-task.png" alt="Task failure error in the Brooklyn debug console." /></a></p> + +<p>By clicking on the <code>stderr</code> link, we can see the script failed with the following error:</p> + +<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">/tmp/brooklyn-20150721-132251052-l4b9-customizing_TomcatServerImpl_i.sh: line 10: mkrid: command not found</span></code></pre></div> + +<p>This tells us <em>what</em> went wrong, but doesnât tell us <em>where</em>. In order to find that, weâll need to look at the +stack trace that was logged when the exception was thrown.</p> + +<p>Itâs always worth looking at the Detailed Status section as sometimes this will give you the information you need. +In this case, the stack trace is limited to the thread that was used to execute the task that ran the script:</p> + +<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">Failed after 40ms</span> + +<span class="go">STDERR</span> +<span class="go">/tmp/brooklyn-20150721-132251052-l4b9-customizing_TomcatServerImpl_i.sh: line 10: mkrid: command not found</span> + + +<span class="go">STDOUT</span> +<span class="go">Executed /tmp/brooklyn-20150721-132251052-l4b9-customizing_TomcatServerImpl_i.sh, result 127: Execution failed, invalid result 127 for customizing TomcatServerImpl{id=e1HP2s8x}</span> + +<span class="go">java.lang.IllegalStateException: Execution failed, invalid result 127 for customizing TomcatServerImpl{id=e1HP2s8x}</span> +<span class="go"> at brooklyn.entity.basic.lifecycle.ScriptHelper.logWithDetailsAndThrow(ScriptHelper.java:390)</span> +<span class="go"> at brooklyn.entity.basic.lifecycle.ScriptHelper.executeInternal(ScriptHelper.java:379)</span> +<span class="go"> at brooklyn.entity.basic.lifecycle.ScriptHelper$8.call(ScriptHelper.java:289)</span> +<span class="go"> at brooklyn.entity.basic.lifecycle.ScriptHelper$8.call(ScriptHelper.java:287)</span> +<span class="go"> at brooklyn.util.task.DynamicSequentialTask$DstJob.call(DynamicSequentialTask.java:343)</span> +<span class="go"> at brooklyn.util.task.BasicExecutionManager$SubmissionCallable.call(BasicExecutionManager.java:469)</span> +<span class="go"> at java.util.concurrent.FutureTask.run(FutureTask.java:262)</span> +<span class="go"> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)</span> +<span class="go"> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)</span> +<span class="go">at java.lang.Thread.run(Thread.java:745)</span></code></pre></div> + +<p>In order to find the exception, weâll need to look in Brooklynâs debug log file. By default, the debug log file +is named <code>brooklyn.debug.log</code>. Usually the easiest way to navigate the log file is to use <code>less</code>, e.g. +<code>less brooklyn.debug.log</code>. We can quickly find find the stack trace by first navigating to the end of the log file +with <code>Shift-G</code>, then performing a reverse-lookup by typing <code>?Tomcat</code> and pressing <code>Enter</code>. If searching for the +blueprint type (in this case Tomcat) simply matches tasks unrelated to the exception, you can also search for +the text of the error message, in this case <code>? invalid result 127</code>. You can make the search case-insensitivity by +typing <code>-i</code> before performing the search. To skip the current match and move to the next on (i.e. âupâ as weâre +performing a reverse-lookup), simply press <code>n</code></p> + +<p>In this case, the <code>?Tomcat</code> search takes us directly to the full stack trace (Only the last part of the trace +is shown here):</p> + +<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">at com.google.common.util.concurrent.ForwardingFuture.get(ForwardingFuture.java:63) ~[guava-17.0.jar:na]</span> +<span class="go">at brooklyn.util.task.BasicTask.get(BasicTask.java:343) ~[classes/:na]</span> +<span class="go">at brooklyn.util.task.BasicTask.getUnchecked(BasicTask.java:352) ~[classes/:na]</span> +<span class="go">... 9 common frames omitted</span> +<span class="go">Caused by: brooklyn.util.exceptions.PropagatedRuntimeException: </span> +<span class="go">at brooklyn.util.exceptions.Exceptions.propagate(Exceptions.java:97) ~[classes/:na]</span> +<span class="go">at brooklyn.util.task.BasicTask.getUnchecked(BasicTask.java:354) ~[classes/:na]</span> +<span class="go">at brooklyn.entity.basic.lifecycle.ScriptHelper.execute(ScriptHelper.java:339) ~[classes/:na]</span> +<span class="go">at brooklyn.entity.webapp.tomcat.TomcatSshDriver.customize(TomcatSshDriver.java:72) ~[classes/:na]</span> +<span class="go">at brooklyn.entity.basic.AbstractSoftwareProcessDriver$8.run(AbstractSoftwareProcessDriver.java:150) ~[classes/:na]</span> +<span class="go">at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) ~[na:1.7.0_71]</span> +<span class="go">at brooklyn.util.task.DynamicSequentialTask$DstJob.call(DynamicSequentialTask.java:343) ~[classes/:na]</span> +<span class="go">... 5 common frames omitted</span> +<span class="go">Caused by: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Execution failed, invalid result 127 for customizing TomcatServerImpl{id=e1HP2s8x}</span> +<span class="go">at java.util.concurrent.FutureTask.report(FutureTask.java:122) [na:1.7.0_71]</span> +<span class="go">at java.util.concurrent.FutureTask.get(FutureTask.java:188) [na:1.7.0_71]</span> +<span class="go">at com.google.common.util.concurrent.ForwardingFuture.get(ForwardingFuture.java:63) ~[guava-17.0.jar:na]</span> +<span class="go">at brooklyn.util.task.BasicTask.get(BasicTask.java:343) ~[classes/:na]</span> +<span class="go">at brooklyn.util.task.BasicTask.getUnchecked(BasicTask.java:352) ~[classes/:na]</span> +<span class="go">... 10 common frames omitted</span> +<span class="go">Caused by: java.lang.IllegalStateException: Execution failed, invalid result 127 for customizing TomcatServerImpl{id=e1HP2s8x}</span> +<span class="go">at brooklyn.entity.basic.lifecycle.ScriptHelper.logWithDetailsAndThrow(ScriptHelper.java:390) ~[classes/:na]</span> +<span class="go">at brooklyn.entity.basic.lifecycle.ScriptHelper.executeInternal(ScriptHelper.java:379) ~[classes/:na]</span> +<span class="go">at brooklyn.entity.basic.lifecycle.ScriptHelper$8.call(ScriptHelper.java:289) ~[classes/:na]</span> +<span class="go">at brooklyn.entity.basic.lifecycle.ScriptHelper$8.call(ScriptHelper.java:287) ~[classes/:na]</span> +<span class="go">... 6 common frames omitted</span></code></pre></div> + +<p>Brooklynâs use of tasks and helper classes can make the stack trace a little harder than usual to follow, but a good +place to start is to look through the stack trace for the nodeâs implementation or ssh driver classes (usually +named <code>FooNodeImpl</code> or <code>FooSshDriver</code>). In this case we can see the following:</p> + +<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">at brooklyn.entity.webapp.tomcat.TomcatSshDriver.customize(TomcatSshDriver.java:72) ~[classes/:na]</span></code></pre></div> + +<p>Combining this with the error message of <code>mkrid: command not found</code> we can see that indeed <code>mkdir</code> has been +misspelled <code>mkrid</code> on line 72 of <code>TomcatSshDriver.java</code>.</p> + +<h2 id="non-script-failure">Non-Script Failure</h2> + +<p>The section above gives an example of a failure that occurs when a script is run. In this section we will look at +a failure in a non-script related part of the code. Weâll use the <code>customize()</code> method of the Tomcat server again, +but this time, weâll correct the spelling of âmkdirâ and add a line that attempts to copy a nonexistent resource +to the remote server:</p> + +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">newScript</span><span class="o">(</span><span class="n">CUSTOMIZING</span><span class="o">)</span> + <span class="o">.</span><span class="na">body</span><span class="o">.</span><span class="na">append</span><span class="o">(</span><span class="s">"mkdir -p conf logs webapps temp"</span><span class="o">)</span> + <span class="o">.</span><span class="na">failOnNonZeroResultCode</span><span class="o">()</span> + <span class="o">.</span><span class="na">execute</span><span class="o">();</span> + +<span class="n">copyTemplate</span><span class="o">(</span><span class="n">entity</span><span class="o">.</span><span class="na">getConfig</span><span class="o">(</span><span class="n">TomcatServer</span><span class="o">.</span><span class="na">SERVER_XML_RESOURCE</span><span class="o">),</span> <span class="n">Os</span><span class="o">.</span><span class="na">mergePaths</span><span class="o">(</span><span class="n">getRunDir</span><span class="o">(),</span> <span class="s">"conf"</span><span class="o">,</span> <span class="s">"server.xml"</span><span class="o">));</span> +<span class="n">copyTemplate</span><span class="o">(</span><span class="n">entity</span><span class="o">.</span><span class="na">getConfig</span><span class="o">(</span><span class="n">TomcatServer</span><span class="o">.</span><span class="na">WEB_XML_RESOURCE</span><span class="o">),</span> <span class="n">Os</span><span class="o">.</span><span class="na">mergePaths</span><span class="o">(</span><span class="n">getRunDir</span><span class="o">(),</span> <span class="s">"conf"</span><span class="o">,</span> <span class="s">"web.xml"</span><span class="o">));</span> +<span class="n">copyTemplate</span><span class="o">(</span><span class="s">"classpath://nonexistent.xml"</span><span class="o">,</span> <span class="n">Os</span><span class="o">.</span><span class="na">mergePaths</span><span class="o">(</span><span class="n">getRunDir</span><span class="o">(),</span> <span class="s">"conf"</span><span class="o">,</span> <span class="s">"nonexistent.xml"</span><span class="o">));</span> <span class="c1">// Resource does not exist!</span></code></pre></div> + +<p>Letâs deploy this using the same YAML from above. Hereâs the resulting error in the Brooklyn debug console:</p> + +<p><a href="images/resource-exception-large.png"><img src="images/resource-exception.png" alt="Resource exception in the Brooklyn debug console." /></a></p> + +<p>Again, this tells us <em>what</em> the error is, but we need to find <em>where</em> the code is that attempts to copy this file. In +this case itâs shown in the Detailed Status section, and we donât need to go to the log file:</p> + +<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">Failed after 221ms: Error getting resource 'classpath://nonexistent.xml' for TomcatServerImpl{id=PVZxDKU1}: java.io.IOException: Error accessing classpath://nonexistent.xml: java.io.IOException: nonexistent.xml not found on classpath</span> + +<span class="go">java.lang.RuntimeException: Error getting resource 'classpath://nonexistent.xml' for TomcatServerImpl{id=PVZxDKU1}: java.io.IOException: Error accessing classpath://nonexistent.xml: java.io.IOException: nonexistent.xml not found on classpath</span> +<span class="go"> at brooklyn.util.ResourceUtils.getResourceFromUrl(ResourceUtils.java:297)</span> +<span class="go"> at brooklyn.util.ResourceUtils.getResourceAsString(ResourceUtils.java:475)</span> +<span class="go"> at brooklyn.entity.basic.AbstractSoftwareProcessDriver.getResourceAsString(AbstractSoftwareProcessDriver.java:447)</span> +<span class="go"> at brooklyn.entity.basic.AbstractSoftwareProcessDriver.processTemplate(AbstractSoftwareProcessDriver.java:469)</span> +<span class="go"> at brooklyn.entity.basic.AbstractSoftwareProcessDriver.copyTemplate(AbstractSoftwareProcessDriver.java:390)</span> +<span class="go"> at brooklyn.entity.basic.AbstractSoftwareProcessDriver.copyTemplate(AbstractSoftwareProcessDriver.java:379)</span> +<span class="go"> at brooklyn.entity.webapp.tomcat.TomcatSshDriver.customize(TomcatSshDriver.java:79)</span> +<span class="go"> at brooklyn.entity.basic.AbstractSoftwareProcessDriver$8.run(AbstractSoftwareProcessDriver.java:150)</span> +<span class="go"> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)</span> +<span class="go"> at brooklyn.util.task.DynamicSequentialTask$DstJob.call(DynamicSequentialTask.java:343)</span> +<span class="go"> at brooklyn.util.task.BasicExecutionManager$SubmissionCallable.call(BasicExecutionManager.java:469)</span> +<span class="go"> at java.util.concurrent.FutureTask.run(FutureTask.java:262)</span> +<span class="go"> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)</span> +<span class="go"> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)</span> +<span class="go">at java.lang.Thread.run(Thread.java:745)</span> +<span class="go"> Caused by: java.io.IOException: Error accessing classpath://nonexistent.xml: java.io.IOException: nonexistent.xml not found on classpath</span> +<span class="go">at brooklyn.util.ResourceUtils.getResourceFromUrl(ResourceUtils.java:233)</span> +<span class="go"> ... 14 more</span> +<span class="go"> Caused by: java.io.IOException: nonexistent.xml not found on classpath</span> +<span class="go"> at brooklyn.util.ResourceUtils.getResourceViaClasspath(ResourceUtils.java:372)</span> +<span class="go">at brooklyn.util.ResourceUtils.getResourceFromUrl(ResourceUtils.java:230)</span> +<span class="go"> ... 14 more</span></code></pre></div> + +<p>Looking for <code>Tomcat</code> in the stack trace, we can see in this case the problem lies at line 79 of <code>TomcatSshDriver.java</code></p> + +<h2 id="external-failure">External Failure</h2> + +<p>Sometimes an entity will fail outside the direct commands issues by Brooklyn. When installing and launching an entity, +Brooklyn will check the return code of scripts that were run to ensure that they completed successfully (i.e. the +return code of the script is zero). It is possible, for example, that a launch script completes successfully, but +the entity fails to start.</p> + +<p>We can simulate this type of failure by launching Tomcat with an invalid configuration file. As seen in the previous +examples, Brooklyn copies two xml configuration files to the server: <code>server.xml</code> and <code>web.xml</code></p> + +<p>The first few non-comment lines of <code>server.xml</code> are as follows (you can see the full file [here] +(https://github.com/apache/incubator-brooklyn/blob/master/software/webapp/src/main/resources/brooklyn/entity/webapp/tomcat/server.xml)):</p> + +<div class="highlight"><pre><code class="language-xml" data-lang="xml"><span class="nt"><Server</span> <span class="na">port=</span><span class="s">"${driver.shutdownPort?c}"</span> <span class="na">shutdown=</span><span class="s">"SHUTDOWN"</span><span class="nt">></span> + <span class="nt"><Listener</span> <span class="na">className=</span><span class="s">"org.apache.catalina.core.AprLifecycleListener"</span> <span class="na">SSLEngine=</span><span class="s">"on"</span> <span class="nt">/></span> + <span class="nt"><Listener</span> <span class="na">className=</span><span class="s">"org.apache.catalina.core.JasperListener"</span> <span class="nt">/></span></code></pre></div> + +<p>Letâs add an unmatched XML element, which will make this XML file invalid:</p> + +<div class="highlight"><pre><code class="language-xml" data-lang="xml"><span class="nt"><Server</span> <span class="na">port=</span><span class="s">"${driver.shutdownPort?c}"</span> <span class="na">shutdown=</span><span class="s">"SHUTDOWN"</span><span class="nt">></span> + <span class="nt"><unmatched-element></span> <span class="c"><!-- This is invalid XML as we won't add </unmatched-element> --></span> + <span class="nt"><Listener</span> <span class="na">className=</span><span class="s">"org.apache.catalina.core.AprLifecycleListener"</span> <span class="na">SSLEngine=</span><span class="s">"on"</span> <span class="nt">/></span> + <span class="nt"><Listener</span> <span class="na">className=</span><span class="s">"org.apache.catalina.core.JasperListener"</span> <span class="nt">/></span></code></pre></div> + +<p>As Brooklyn doesnât know how these types of resources are used, theyâre not validated as theyâre copied to the remote machine. +As far as Brooklyn is concerned, the file will have copied successfully.</p> + +<p>Letâs deploy Tomcat again, using the same YAML as before. This time, the deployment runs for a few minutes before failing +with <code>Timeout waiting for SERVICE_UP</code>:</p> + +<p><a href="images/external-error-large.png"><img src="images/external-error.png" alt="External error in the Brooklyn debug console." /></a></p> + +<p>If we drill down into the tasks in the <code>Activities</code> tab, we can see that all of the installation and launch tasks +completed successfully, and stdout of the <code>launch</code> script is as follows:</p> + +<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">Executed /tmp/brooklyn-20150721-153049139-fK2U-launching_TomcatServerImpl_id_.sh, result 0</span></code></pre></div> + +<p>The task that failed was the <code>post-start</code> task, and the stack trace from the Detailed Status section is as follows:</p> + +<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">Failed after 5m 1s: Timeout waiting for SERVICE_UP from TomcatServerImpl{id=BUHgQeOs}</span> + +<span class="go">java.lang.IllegalStateException: Timeout waiting for SERVICE_UP from TomcatServerImpl{id=BUHgQeOs}</span> +<span class="go"> at brooklyn.entity.basic.Entities.waitForServiceUp(Entities.java:1073)</span> +<span class="go"> at brooklyn.entity.basic.SoftwareProcessImpl.waitForServiceUp(SoftwareProcessImpl.java:388)</span> +<span class="go"> at brooklyn.entity.basic.SoftwareProcessImpl.waitForServiceUp(SoftwareProcessImpl.java:385)</span> +<span class="go"> at brooklyn.entity.basic.SoftwareProcessDriverLifecycleEffectorTasks.postStartCustom(SoftwareProcessDriverLifecycleEffectorTasks.java:164)</span> +<span class="go"> at brooklyn.entity.software.MachineLifecycleEffectorTasks$7.run(MachineLifecycleEffectorTasks.java:433)</span> +<span class="go"> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)</span> +<span class="go"> at brooklyn.util.task.DynamicSequentialTask$DstJob.call(DynamicSequentialTask.java:343)</span> +<span class="go"> at brooklyn.util.task.BasicExecutionManager$SubmissionCallable.call(BasicExecutionManager.java:469)</span> +<span class="go"> at java.util.concurrent.FutureTask.run(FutureTask.java:262)</span> +<span class="go"> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)</span> +<span class="go"> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)</span> +<span class="go">at java.lang.Thread.run(Thread.java:745)</span></code></pre></div> + +<p>This doesnât really tell us what we need to know, and looking in the <code>brooklyn.debug.log</code> file yields no further +clues. The key here is the error message <code>Timeout waiting for SERVICE_UP</code>. After running the installation and +launch scripts, assuming all scripts completed successfully, Brooklyn will periodically check the health of the node +and will set the node on fire if the health check does not pass within a pre-prescribed period (the default is +two minutes, and can be configured using the <code>start.timeout</code> config key). The periodic health check also continues +after the successful launch in order to check continued operation of the node, but in this case it fails to pass +at all.</p> + +<p>The first thing we need to do is to find out how Brooklyn determines the health of the node. The health-check is +often implemented in the <code>isRunning()</code> method in the entityâs ssh driver. Tomcatâs implementation of <code>isRunning()</code> +is as follows:</p> + +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="nd">@Override</span> +<span class="kd">public</span> <span class="kt">boolean</span> <span class="nf">isRunning</span><span class="o">()</span> <span class="o">{</span> + <span class="k">return</span> <span class="nf">newScript</span><span class="o">(</span><span class="n">MutableMap</span><span class="o">.</span><span class="na">of</span><span class="o">(</span><span class="n">USE_PID_FILE</span><span class="o">,</span> <span class="s">"pid.txt"</span><span class="o">),</span> <span class="n">CHECK_RUNNING</span><span class="o">).</span><span class="na">execute</span><span class="o">()</span> <span class="o">==</span> <span class="mi">0</span><span class="o">;</span> +<span class="o">}</span></code></pre></div> + +<p>The <code>newScript</code> method has conveniences for default scripts to check if a process is running based on its PID. In this +case, it will look for Tomcatâs PID in the <code>pid.txt</code> file and check if the PID is the PID of a running process</p> + +<p>Itâs worth a quick sanity check at this point to check if the PID file exists, and if the process is running. +By default, the pid file is located in the run directory of the entity. You can find the location of the entityâs run +directory by looking at the <code>run.dir</code> sensor. In this case it is <code>/tmp/brooklyn-martin/apps/jIzIHXtP/entities/TomcatServer_BUHgQeOs</code>. +To find the pid, you simply cat the pid.txt file in this directory:</p> + +<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="gp">$</span> cat /tmp/brooklyn-martin/apps/jIzIHXtP/entities/TomcatServer_BUHgQeOs/pid.txt +<span class="go">73714</span></code></pre></div> + +<p>In this case, the PID in the file is 73714. You can then check if the process is running using <code>ps</code>. You can also +pipe the output to <code>fold</code> so the full launch command is visible:</p> + +<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="gp">$</span> ps -p <span class="m">73714</span> <span class="p">|</span> fold -w 120 +<span class="go">PID TTY TIME CMD</span> +<span class="go">73714 ?? 0:08.03 /Library/Java/JavaVirtualMachines/jdk1.8.0_51.jdk/Contents/Home/bin/java -Dnop -Djava.util.logg</span> +<span class="go">ing.manager=org.apache.juli.ClassLoaderLogManager -javaagent:/tmp/brooklyn-martin/apps/jIzIHXtP/entities/TomcatServer_BU</span> +<span class="go">HgQeOs/brooklyn-jmxmp-agent-shaded-0.8.0-SNAPSHOT.jar -Xms200m -Xmx800m -XX:MaxPermSize=400m -Dcom.sun.management.jmxrem</span> +<span class="go">ote -Dbrooklyn.jmxmp.rmi-port=1099 -Dbrooklyn.jmxmp.port=31001 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.manage</span> +<span class="go">ment.jmxremote.authenticate=false -Djava.endorsed.dirs=/tmp/brooklyn-martin/installs/TomcatServer_7.0.56/apache-tomcat-7</span> +<span class="go">.0.56/endorsed -classpath /tmp/brooklyn-martin/installs/TomcatServer_7.0.56/apache-tomcat-7.0.56/bin/bootstrap.jar:/tmp/</span> +<span class="go">brooklyn-martin/installs/TomcatServer_7.0.56/apache-tomcat-7.0.56/bin/tomcat-juli.jar -Dcatalina.base=/tmp/brooklyn-mart</span> +<span class="go">in/apps/jIzIHXtP/entities/TomcatServer_BUHgQeOs -Dcatalina.home=/tmp/brooklyn-martin/installs/TomcatServer_7.0.56/apache</span> +<span class="go">-tomcat-7.0.56 -Djava.io.tmpdir=/tmp/brooklyn-martin/apps/jIzIHXtP/entities/TomcatServer_BUHgQeOs/temp org.apache.catali</span> +<span class="go">na.startup.Bootstrap start</span></code></pre></div> + +<p>This confirms that the process is running. The next thing we can look at is the <code>service.notUp.indicators</code> sensor. This +reads as follows:</p> + +<div class="highlight"><pre><code class="language-json" data-lang="json"><span class="p">{</span><span class="nt">"service.process.isRunning"</span><span class="p">:</span><span class="s2">"The software process for this entity does not appear to be running"</span><span class="p">}</span></code></pre></div> + +<p>This confirms that the problem is indeed due to the <code>service.process.isRunning</code> sensor. We assumed earlier that this was +set by the <code>isRunning()</code> method in <code>TomcatSshDriver.java</code>, but this isnât always the case. The <code>service.process.isRunning</code> +sensor is wired up by the <code>connectSensors()</code> method in the nodeâs implementation class, in this case +<code>TomcatServerImpl.java</code>. Tomcatâs implementation of <code>connectSensors()</code> is as follows:</p> + +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="nd">@Override</span> +<span class="kd">public</span> <span class="kt">void</span> <span class="nf">connectSensors</span><span class="o">()</span> <span class="o">{</span> + <span class="kd">super</span><span class="o">.</span><span class="na">connectSensors</span><span class="o">();</span> + + <span class="k">if</span> <span class="o">(</span><span class="n">getDriver</span><span class="o">().</span><span class="na">isJmxEnabled</span><span class="o">())</span> <span class="o">{</span> + <span class="n">String</span> <span class="n">requestProcessorMbeanName</span> <span class="o">=</span> <span class="s">"Catalina:type=GlobalRequestProcessor,name=\"http-*\""</span><span class="o">;</span> + + <span class="n">Integer</span> <span class="n">port</span> <span class="o">=</span> <span class="n">isHttpsEnabled</span><span class="o">()</span> <span class="o">?</span> <span class="n">getAttribute</span><span class="o">(</span><span class="n">HTTPS_PORT</span><span class="o">)</span> <span class="o">:</span> <span class="n">getAttribute</span><span class="o">(</span><span class="n">HTTP_PORT</span><span class="o">);</span> + <span class="n">String</span> <span class="n">connectorMbeanName</span> <span class="o">=</span> <span class="n">format</span><span class="o">(</span><span class="s">"Catalina:type=Connector,port=%s"</span><span class="o">,</span> <span class="n">port</span><span class="o">);</span> + + <span class="n">jmxWebFeed</span> <span class="o">=</span> <span class="n">JmxFeed</span><span class="o">.</span><span class="na">builder</span><span class="o">()</span> + <span class="o">.</span><span class="na">entity</span><span class="o">(</span><span class="k">this</span><span class="o">)</span> + <span class="o">.</span><span class="na">period</span><span class="o">(</span><span class="mi">3000</span><span class="o">,</span> <span class="n">TimeUnit</span><span class="o">.</span><span class="na">MILLISECONDS</span><span class="o">)</span> + <span class="o">.</span><span class="na">pollAttribute</span><span class="o">(</span><span class="k">new</span> <span class="n">JmxAttributePollConfig</span><span class="o"><</span><span class="n">Integer</span><span class="o">>(</span><span class="n">ERROR_COUNT</span><span class="o">)</span> + <span class="o">.</span><span class="na">objectName</span><span class="o">(</span><span class="n">requestProcessorMbeanName</span><span class="o">)</span> + <span class="o">.</span><span class="na">attributeName</span><span class="o">(</span><span class="s">"errorCount"</span><span class="o">))</span> + <span class="o">.</span><span class="na">pollAttribute</span><span class="o">(</span><span class="k">new</span> <span class="n">JmxAttributePollConfig</span><span class="o"><</span><span class="n">Integer</span><span class="o">>(</span><span class="n">REQUEST_COUNT</span><span class="o">)</span> + <span class="o">.</span><span class="na">objectName</span><span class="o">(</span><span class="n">requestProcessorMbeanName</span><span class="o">)</span> + <span class="o">.</span><span class="na">attributeName</span><span class="o">(</span><span class="s">"requestCount"</span><span class="o">))</span> + <span class="o">.</span><span class="na">pollAttribute</span><span class="o">(</span><span class="k">new</span> <span class="n">JmxAttributePollConfig</span><span class="o"><</span><span class="n">Integer</span><span class="o">>(</span><span class="n">TOTAL_PROCESSING_TIME</span><span class="o">)</span> + <span class="o">.</span><span class="na">objectName</span><span class="o">(</span><span class="n">requestProcessorMbeanName</span><span class="o">)</span> + <span class="o">.</span><span class="na">attributeName</span><span class="o">(</span><span class="s">"processingTime"</span><span class="o">))</span> + <span class="o">.</span><span class="na">pollAttribute</span><span class="o">(</span><span class="k">new</span> <span class="n">JmxAttributePollConfig</span><span class="o"><</span><span class="n">String</span><span class="o">>(</span><span class="n">CONNECTOR_STATUS</span><span class="o">)</span> + <span class="o">.</span><span class="na">objectName</span><span class="o">(</span><span class="n">connectorMbeanName</span><span class="o">)</span> + <span class="o">.</span><span class="na">attributeName</span><span class="o">(</span><span class="s">"stateName"</span><span class="o">))</span> + <span class="o">.</span><span class="na">pollAttribute</span><span class="o">(</span><span class="k">new</span> <span class="n">JmxAttributePollConfig</span><span class="o"><</span><span class="n">Boolean</span><span class="o">>(</span><span class="n">SERVICE_PROCESS_IS_RUNNING</span><span class="o">)</span> + <span class="o">.</span><span class="na">objectName</span><span class="o">(</span><span class="n">connectorMbeanName</span><span class="o">)</span> + <span class="o">.</span><span class="na">attributeName</span><span class="o">(</span><span class="s">"stateName"</span><span class="o">)</span> + <span class="o">.</span><span class="na">onSuccess</span><span class="o">(</span><span class="n">Functions</span><span class="o">.</span><span class="na">forPredicate</span><span class="o">(</span><span class="n">Predicates</span><span class="o">.<</span><span class="n">Object</span><span class="o">></span><span class="n">equalTo</span><span class="o">(</span><span class="s">"STARTED"</span><span class="o">)))</span> + <span class="o">.</span><span class="na">setOnFailureOrException</span><span class="o">(</span><span class="kc">false</span><span class="o">))</span> + <span class="o">.</span><span class="na">build</span><span class="o">();</span> + + <span class="n">jmxAppFeed</span> <span class="o">=</span> <span class="n">JavaAppUtils</span><span class="o">.</span><span class="na">connectMXBeanSensors</span><span class="o">(</span><span class="k">this</span><span class="o">);</span> + <span class="o">}</span> <span class="k">else</span> <span class="o">{</span> + <span class="c1">// if not using JMX</span> + <span class="n">LOG</span><span class="o">.</span><span class="na">warn</span><span class="o">(</span><span class="s">"Tomcat running without JMX monitoring; limited visibility of service available"</span><span class="o">);</span> + <span class="n">connectServiceUpIsRunning</span><span class="o">();</span> + <span class="o">}</span> +<span class="o">}</span></code></pre></div> + +<p>We can see here that if jmx is not enabled, the method will call <code>connectServiceUpIsRunning()</code> which will use the +default PID-based method of determining if a process is running. However, as JMX <em>is</em> running, the <code>service.process.isRunning</code> +sensor (denoted here by the <code>SERVICE_PROCESS_IS_RUNNING</code> variable) is set to true if and only if the +<code>stateName</code> JMX attribute equals <code>STARTED</code>. We can see from the previous call to <code>.pollAttribute</code> that this +attribute is also published to the <code>CONNECTOR_STATUS</code> sensor. The <code>CONNECTOR_STATUS</code> sensor is defined as follows:</p> + +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">AttributeSensor</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">CONNECTOR_STATUS</span> <span class="o">=</span> + <span class="k">new</span> <span class="n">BasicAttributeSensor</span><span class="o"><</span><span class="n">String</span><span class="o">>(</span><span class="n">String</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="s">"webapp.tomcat.connectorStatus"</span><span class="o">,</span> <span class="s">"Catalina connector state name"</span><span class="o">);</span></code></pre></div> + +<p>Letâs go back to the Brooklyn debug console and look for the <code>webapp.tomcat.connectorStatus</code>:</p> + +<p><a href="images/jmx-sensors-large.png"><img src="images/jmx-sensors.png" alt="Sensors view in the Brooklyn debug console." /></a></p> + +<p>As the sensor is not shown, itâs likely that itâs simply null or not set. We can check this by clicking +the <code>Show/hide empty records</code> icon (highlighted in yellow above):</p> + +<p><a href="images/jmx-sensors-all-large.png"><img src="images/jmx-sensors-all.png" alt="All sensors view in the Brooklyn debug console." /></a></p> + +<p>We know from previous steps that the installation and launch scripts completed, and we know the procecess is running, +but we can see here that the server is not responding to JMX requests. A good thing to check here would be that the +JMX port is not being blocked by iptables, firewalls or security groups +(see the (troubleshooting connectivity guide)[troubleshooting-connectivity.html]). +Letâs assume that weâve checked that and theyâre all open. There is still one more thing that Brooklyn can tell us.</p> + +<p>Still on the <code>Sensors</code> tab, letâs take a look at the <code>log.location</code> sensor:</p> + +<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">/tmp/brooklyn-martin/apps/c3bmrlC3/entities/TomcatServer_C1TAjYia/logs/catalina.out</span></code></pre></div> + +<p>This is the location of Tomcatâs own log file. The location of the log file will differ from process to process +and when writing a custom entity you will need to check the softwareâs own documentation. If your blueprintâs +ssh driver extends <code>JavaSoftwareProcessSshDriver</code>, the value returned by the <code>getLogFileLocation()</code> method will +automatically be published to the <code>log.location</code> sensor. Otherwise, you can publish the value yourself by calling +<code>entity.setAttribute(Attributes.LOG_FILE_LOCATION, getLogFileLocation());</code> in your ssh driver</p> + +<p><strong>Note:</strong> The log file will be on the server to which you have deployed Tomcat, and not on the Brooklyn server. +Letâs take a look in the log file:</p> + +<div class="highlight"><pre><code class="language-console" data-lang="console"><span class="go">less /tmp/brooklyn-martin/apps/c3bmrlC3/entities/TomcatServer_C1TAjYia/logs/catalina.out</span> + +<span class="go">Jul 21, 2015 4:12:12 PM org.apache.tomcat.util.digester.Digester fatalError</span> +<span class="go">SEVERE: Parse Fatal Error at line 143 column 3: The element type "unmatched-element" must be terminated by the matching end-tag "</unmatched-element>".</span> +<span class="go"> org.xml.sax.SAXParseException; systemId: file:/tmp/brooklyn-martin/apps/c3bmrlC3/entities/TomcatServer_C1TAjYia/conf/server.xml; lineNumber: 143; columnNumber: 3; The element type "unmatched-element" must be terminated by the matching end-tag "</unmatched-element>".</span> +<span class="go"> at com.sun.org.apache.xerces.internal.util.ErrorHandlerWrapper.createSAXParseException(ErrorHandlerWrapper.java:203)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.util.ErrorHandlerWrapper.fatalError(ErrorHandlerWrapper.java:177)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLErrorReporter.reportError(XMLErrorReporter.java:441)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLErrorReporter.reportError(XMLErrorReporter.java:368)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLScanner.reportFatalError(XMLScanner.java:1437)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl.scanEndElement(XMLDocumentFragmentScannerImpl.java:1749)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl$FragmentContentDriver.next(XMLDocumentFragmentScannerImpl.java:2973)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLDocumentScannerImpl.next(XMLDocumentScannerImpl.java:606)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl.scanDocument(XMLDocumentFragmentScannerImpl.java:510)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.parsers.XML11Configuration.parse(XML11Configuration.java:848)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.parsers.XML11Configuration.parse(XML11Configuration.java:777)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.parsers.XMLParser.parse(XMLParser.java:141)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.parsers.AbstractSAXParser.parse(AbstractSAXParser.java:1213)</span> +<span class="go"> at com.sun.org.apache.xerces.internal.jaxp.SAXParserImpl$JAXPSAXParser.parse(SAXParserImpl.java:649)</span> +<span class="go"> at org.apache.tomcat.util.digester.Digester.parse(Digester.java:1561)</span> +<span class="go"> at org.apache.catalina.startup.Catalina.load(Catalina.java:615)</span> +<span class="go"> at org.apache.catalina.startup.Catalina.start(Catalina.java:677)</span> +<span class="go"> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)</span> +<span class="go"> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)</span> +<span class="go"> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)</span> +<span class="go"> at java.lang.reflect.Method.invoke(Method.java:497)</span> +<span class="go"> at org.apache.catalina.startup.Bootstrap.start(Bootstrap.java:321)</span> +<span class="go">at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:455)</span> + +<span class="go"> Jul 21, 2015 4:12:12 PM org.apache.catalina.startup.Catalina load</span> +<span class="go"> WARNING: Catalina.start using conf/server.xml: The element type "unmatched-element" must be terminated by the matching end-tag "</unmatched-element>".</span> +<span class="go"> Jul 21, 2015 4:12:12 PM org.apache.catalina.startup.Catalina start</span> +<span class="go"> SEVERE: Cannot start server. Server instance is not configured.</span></code></pre></div> + +<p>As expected, we can see here that the <code>unmatched-element</code> element has not been terminated in the <code>server.xml</code> file</p> + + </div> + + <div class="col-md-3"> + <div class="list-group side-menu" id="side-menu"> + + + + + + + <h4 class=" with_following"> + <a href="/v/latest/index.html" class="list-group-item breadcrumb breadcrumb0"> + User Guide + </a></h4> + + <h4 class=" with_preceding with_following"> + <a href="/v/latest/ops/index.html" class="list-group-item breadcrumb breadcrumb1"> + Operations + </a></h4> + + <h4 class=" with_preceding"> + <a href="/v/latest/ops/troubleshooting/index.html" class="list-group-item breadcrumb breadcrumb2"> + Troubleshooting + </a></h4> + + + + + + <a href="/v/latest/ops/troubleshooting/overview.html" class="list-group-item">Overview + </a> + + + + + + <a href="/v/latest/ops/troubleshooting/deployment.html" class="list-group-item">Deployment + </a> + + + + + + <a href="/v/latest/ops/troubleshooting/connectivity.html" class="list-group-item">Server Connectivity + </a> + + + + + + <a href="/v/latest/ops/troubleshooting/softwareprocess.html" class="list-group-item">SoftwareProcess Entities + </a> + + + + + + <a href="/v/latest/ops/troubleshooting/going-deep-in-java-and-logs.html" class="list-group-item active">Going Deep in Java and Logs + </a> + + + + + + + +</div> +<div id="width_reference"></div> + + +<script language="JavaScript" type="application/javascript"> + + +sidemenu_x_sizer=function(){ $('#side-menu').width($('#side-menu').parent().find('#width_reference').outerWidth()); }; +$(sidemenu_x_sizer); +$(window).resize(sidemenu_x_sizer); + + +sidemenu_y_nonfloater=function(){ + if ($('#side-menu').outerHeight(true) + $('#header').outerHeight(true) + $('#footer').outerHeight(true) > window.innerHeight || + $('#side-menu').width() >= $('#content_container').width()/2) { + $('#side-menu').css('position', 'inherit'); + } else { + // restore if screen has grown + $('#side-menu').css('position', 'fixed'); + } +}; +$(sidemenu_y_nonfloater); +$(window).resize(sidemenu_y_nonfloater); + + + +var sideMenu = $("#side-menu"), + sideItems = sideMenu.find("a"), + // Anchors corresponding to menu items + scrollItems = sideItems.map(function(){ + var item = $(this).attr("section-target"); + if (item && item.length) { return item; } + }); + +var highlight_section_last_top = -1; +var highlight_section_completed = false; + +var highlight_section = function() { + // Get container scroll position + var highlight_section_new_top = $(this).scrollTop(); + if (highlight_section_new_top == highlight_section_last_top) return; + var highlight_section_new_bottom = highlight_section_new_top + $(window).height(); + var scroll_advancing = (highlight_section_new_top > highlight_section_last_top); + + var last_item = null, active_item = $("#side-menu a.section#active"); + + var found_top = false; + var displayable_items = scrollItems.map(function(itemI){ + item = $(scrollItems[itemI]); + if (item && item.length) { + if (highlight_section_last_top == -1 || !highlight_section_completed) { + // just opening page - take item matching hash, or otherwise the first item visible + if (item.selector === window.location.hash || (item.offset().top > highlight_section_new_top - 20 && !found_top)) { + found_top = true; + if (item.selector === window.location.hash && item.offset().top < highlight_section_new_top + 60) { + // because of our top header, we need to scroll 64px down from any link + $('html, body').animate({scrollTop: item.offset().top - 64}, 0); + } + return item; + } + } else if (scroll_advancing) { + // if scrolling advance, pick up a section when title starts before 1/3 height + if (item.offset().top < highlight_section_new_top + $(window).height()/3) + return item; + + // or if containing div is finished (usu the whole main content) + div_containing_item = item.closest("div"); + if (div_containing_item.offset().top + div_containing_item.height() < highlight_section_new_bottom + 15) + return item; + // or when next title is visible + if (last_item && item.offset().top < highlight_section_new_bottom + 15) + return last_item; + } else { + // if scrolling back, pick up a section as soon as the title is visible, + if (item.offset().top < highlight_section_new_top) + return item; + // or if title is before the 2/3 point + // (not sure about this, probably want also to have + // "AND the id.top is <= displayable_itemsrent_active_it.top" so we don't jump FORWARD a section + // when scrolling BACK, with lots of tiny sections) + if ((item.offset().top < highlight_section_new_top + 2*$(window).height()/3) + && (!active_item || !active_item.offset() || active_item.offset().top >= item.offset().top)) + return item; + + } + last_item = item; + } + }); + if (!highlight_section_completed && document.readyState === "complete") { + highlight_section_completed = true; + } + if (!displayable_items.length) { + $("#side-menu a.section").removeClass("active"); + } else { + displayable_items = displayable_items[displayable_items.length-1]; + var id = displayable_items && displayable_items.length ? displayable_items[0].id : ""; + // Set/remove active class + new_active = $("#side-menu a.section").filter("[section-target='#"+id+"']"); + if (new_active.hasClass("active")) { + // nothing needed + } else { + $("#side-menu a.section").removeClass("active"); + $("#side-menu a.section").filter("[section-target='#"+id+"']").addClass("active"); + } + } + + highlight_section_last_top = highlight_section_new_top; +}; +var highlight_new_section = function() { + highlight_section_completed = false; + highlight_section_last_top = -1; + highlight_section(); +} + +$(window).scroll(highlight_section); +$(highlight_new_section); + +// detect link change - courtesy http://www.bennadel.com/blog/1520-binding-events-to-non-dom-objects-with-jquery.htm + ( + function( $ ){ + // Default to the current location. + var strLocation = window.location.href; + var strHash = window.location.hash; + var strPrevLocation = ""; + var strPrevHash = ""; + + // This is how often we will be checkint for + // changes on the location. + var intIntervalTime = 100; + + // This method removes the pound from the hash. + var fnCleanHash = function( strHash ){ + return( + strHash.substring( 1, strHash.length ) + ); + } + + // This will be the method that we use to check + // changes in the window location. + var fnCheckLocation = function(){ + // Check to see if the location has changed. + if (strLocation != window.location.href){ + + // Store the new and previous locations. + strPrevLocation = strLocation; + strPrevHash = strHash; + strLocation = window.location.href; + strHash = window.location.hash; + + // The location has changed. Trigger a + // change event on the location object, + // passing in the current and previous + // location values. + $( window.location ).trigger( + "change", + { + currentHref: strLocation, + currentHash: fnCleanHash( strHash ), + previousHref: strPrevLocation, + previousHash: fnCleanHash( strPrevHash ) + } + ); + + } + } + + // Set an interval to check the location changes. + setInterval( fnCheckLocation, intIntervalTime ); + } + )( jQuery ); +// and trigger highlight section on link change +$(window.location).bind("change", highlight_new_section); + +</script> + + </div> + </div> +</div> + +<div id="footer"> + <div class="container"> + <div class="row"> + <div class="col-md-10 text-muted"> + Apache Brooklyn is distributed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache + License v2.0</a>. + <br /> + Apache Brooklyn is currently undergoing Incubation at The Apache Software + Foundation. + </div> + <div class="col-md-2"> + <a class="btn btn-sm btn-default" href="https://github.com/apache/incubator-brooklyn/edit/master/docs/guide/ops/troubleshooting/going-deep-in-java-and-logs.md";>Edit This Page</a> + <a href="http://0.0.0.0:4000/community/how-to-contribute-docs.html"; + data-toggle="tooltip" data-placement="top" title="How to Edit Documentation" data-delay="400"/> + <span class="octicon octicon-question octicon-footer"></span> + </a> + </div> + </div> + </div> +</div> + + + + + +<script src="//cdnjs.cloudflare.com/ajax/libs/zeroclipboard/1.3.1/ZeroClipboard.min.js"></script> + +<script language="JavaScript" type="application/javascript"> + fix_padding_function = function () { + $('body').css('padding-top', parseInt($('#header').css("height"))+10); + $('body').css('padding-bottom', parseInt($('#footer').css("height"))+10); + }; + $(window).resize(fix_padding_function); + $(window).load(fix_padding_function); + + $(function () { + $('[data-toggle="tooltip"]').tooltip({ delay: { show: 600, hide: 100 }}) + }); + +/* generate anchors for headers, a la github and http://blog.parkermoore.de/2014/08/01/header-anchor-links-in-vanilla-javascript-for-github-pages-and-jekyll/ */ +var anchorForId = function (id, text) { + var anchor = document.createElement("a"); + anchor.className = "header-link"; + anchor.href = "#" + id; + anchor.innerHTML = "<i class=\"fa fa-link\"></i>"; + return anchor; +}; + +var linkifyAnchors = function (level, containingElement) { + var headers = contentBlock.find("h" + level); + for (var h = 0; h < headers.length; h++) { + var header = headers[h]; + if (typeof header.id !== "undefined" && header.id !== "") { + header.appendChild(anchorForId(header.id, $(header).text())); + } + } +}; + +$(function () { + contentBlock = $("#content_container"); + if (!contentBlock) return; + for (var level = 1; level <= 6; level++) { + linkifyAnchors(level, contentBlock); + } +}); + +<!-- Clipboard support --> + ZeroClipboard.config({ moviePath: '//cdnjs.cloudflare.com/ajax/libs/zeroclipboard/1.3.1/ZeroClipboard.swf' }); + +$(function() { + $('div.highlight').prepend( + $('<div class="clipboard_container" title="Copy to Clipboard">'+ + '<div class="fa clipboard_button">'+ + '<div class="on-active"><div>Copied to Clipboard</div></div>'+ + '</div></div>')); + $('div.clipboard_container').each(function(index) { + var clipboard = new ZeroClipboard(); + clipboard.clip( $(this).find(":first")[0], $(this)[0] ); + var target = $(this).next(); + var txt = target.text().trim(); + if (target.find('code.bash')) { + // Strip out bash prompts from the start of each line (i.e. '$' or '%' characters + // at the very start, or immediately following any newline). Correctly handles continuation + // lines, where a leading '$' or '%' is *not* a prompt character. + txt = txt.replace(/(^|[^\\]\n)[$%] /g, "$1"); + } + clipboard.on( 'dataRequested', function (client, args) { + client.setText( txt ); + }); + }); +}); + +<!-- search --> + $(function() { + $('#simple_google') + .submit(function() { + $('input[name="q"]').val("site:" + document.location.hostname + " " + $('input[name="brooklyn-search"]').val()); + return true; + }); + $('input[name="brooklyn-search"]').focus(function() { + if ($(this).val() === $(this).attr('placeholder')) { + $(this).val(''); + } + }) + .blur(function() { + if ($(this).val() === '') { + $(this).val($(this).attr('placeholder')); + } + }) + .blur(); + }); + + + <!-- analytics --> + var _gaq = _gaq || []; + _gaq.push(['_setAccount', 'UA-30530918-1']); + _gaq.push(['_trackPageview']); + + (function() { + var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; + ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; + var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); + })(); + + +<!-- page warning (for archive pages) --> + + + $(function() { + if (document.location.pathname.replace(/^\/([^\/]*).*$/, '$1') === "v"){ + var thisversion = document.location.pathname.split("/")[2], + msg = ""; + if (thisversion != 'latest' && (!$.cookie('brooklyn_versions') || + (($.inArray('ALL', $.cookie('brooklyn_versions').split(",")) === -1) && + ($.inArray(thisversion, $.cookie('brooklyn_versions').split(",")) === -1))) ){ + msg += "<div class='warning_banner_image'><img src='/v/latest/style/img/warning.png'/></div>"; + msg += "<p>You are browsing the archive site for version <strong>"+thisversion+"</strong>.</p>"; + if (thisversion.indexOf("SNAPSHOT") >= 0) { + msg += "<p>Note that SNAPSHOT versions have not been voted on and are not endorsed by the Apache Software Foundation.</p>"; + msg += "<p>Do you understand and accept the risks?</p>"; + } else { + msg += "<p>Is this deliberate?</p>"; + } + msg += "<center><p class='warning_banner_buttons'>"; + msg += "<a href = 'javascript:void(0);' onclick=\"set_user_version('"+thisversion+"');\">Yes, hide this warning</a>"; + msg += "<a href = '/v/latest/'>No, take me to the latest version guide</a>"; + msg += "<a href = '/meta/versions.html'>Show all versions</a>"; + msg += "</p></center>" + + $('#page_notes').html(msg); + $('#page_notes').fadeIn('slow'); + } + } + }); + + function get_user_versions() { + return $.cookie("brooklyn_versions") ? $.cookie("brooklyn_versions").split(",") : []; + }; + function set_user_version(version) { + var version_cookie = get_user_versions(); + version_cookie.push(version); + $.cookie('brooklyn_versions', version_cookie, { expires: 365, path: '/' }); + $('#page_notes').fadeOut(); + event.preventDefault ? event.preventDefault() : event.returnValue = false; + }; + function set_user_versions_all() { + var version_cookie = get_user_versions(); + version_cookie.push("ALL"); + $.cookie('brooklyn_versions', version_cookie, { expires: 365, path: '/' }); + $('#page_notes').fadeOut(); + event.preventDefault ? event.preventDefault() : event.returnValue = false; + }; + function clear_user_versions() { + $.removeCookie('brooklyn_versions', { path: '/' }); + $('#page_notes').fadeIn('slow'); + event.preventDefault ? event.preventDefault() : event.returnValue = false; + }; + +</script> + +</body> + +</html>
Added: incubator/brooklyn/site/v/latest/ops/troubleshooting/images/failed-task-large.png URL: http://svn.apache.org/viewvc/incubator/brooklyn/site/v/latest/ops/troubleshooting/images/failed-task-large.png?rev=1693583&view=auto ============================================================================== Binary file - no diff available. Propchange: incubator/brooklyn/site/v/latest/ops/troubleshooting/images/failed-task-large.png ------------------------------------------------------------------------------ svn:mime-type = image/png Added: incubator/brooklyn/site/v/latest/ops/troubleshooting/images/jmx-sensors-large.png URL: http://svn.apache.org/viewvc/incubator/brooklyn/site/v/latest/ops/troubleshooting/images/jmx-sensors-large.png?rev=1693583&view=auto ============================================================================== Binary file - no diff available. Propchange: incubator/brooklyn/site/v/latest/ops/troubleshooting/images/jmx-sensors-large.png ------------------------------------------------------------------------------ svn:mime-type = image/png
