DRILL-4571: Add link to local Drill logs from the web UI This closes #472
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1a89a7fe Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1a89a7fe Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1a89a7fe Branch: refs/heads/master Commit: 1a89a7fe56533ecea8f7a7c9be6a3699925f3c96 Parents: 94b8aec Author: Arina Ielchiieva <arina.yelchiy...@gmail.com> Authored: Thu Mar 31 18:43:25 2016 +0300 Committer: Parth Chandra <par...@apache.org> Committed: Tue May 3 10:50:09 2016 -0700 ---------------------------------------------------------------------- distribution/src/resources/logback.xml | 2 +- .../org/apache/drill/exec/ExecConstants.java | 5 + .../server/options/SystemOptionManager.java | 3 +- .../drill/exec/server/rest/DrillRestServer.java | 1 + .../drill/exec/server/rest/LogsResources.java | 211 +++++++++++++++++++ .../server/rest/ViewableWithPermissions.java | 1 + .../src/main/resources/rest/generic.ftl | 3 + .../src/main/resources/rest/logs/list.ftl | 55 +++++ .../src/main/resources/rest/logs/log.ftl | 37 ++++ 9 files changed, 316 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/distribution/src/resources/logback.xml ---------------------------------------------------------------------- diff --git a/distribution/src/resources/logback.xml b/distribution/src/resources/logback.xml index 350383a..fb53dfc 100644 --- a/distribution/src/resources/logback.xml +++ b/distribution/src/resources/logback.xml @@ -44,7 +44,7 @@ <maxFileSize>100MB</maxFileSize> </triggeringPolicy> <encoder> - <pattern>%msg</pattern> + <pattern>%msg%n</pattern> </encoder> </appender> http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index 7f216f0..17fbb7b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -303,4 +303,9 @@ public interface ExecConstants { StringValidator IMPERSONATION_POLICY_VALIDATOR = new InboundImpersonationManager.InboundImpersonationPolicyValidator(IMPERSONATION_POLICIES_KEY, "[]"); + /** + * Web settings + */ + String WEB_LOGS_MAX_LINES = "web.logs.max_lines"; + OptionValidator WEB_LOGS_MAX_LINES_VALIDATOR = new PositiveLongValidator(WEB_LOGS_MAX_LINES, Integer.MAX_VALUE, 10000); } http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index db78108..c35ed0e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -135,7 +135,8 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea ExecConstants.ENABLE_WINDOW_FUNCTIONS_VALIDATOR, ClassTransformer.SCALAR_REPLACEMENT_VALIDATOR, ExecConstants.ENABLE_NEW_TEXT_READER, - ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST + ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST, + ExecConstants.WEB_LOGS_MAX_LINES_VALIDATOR }; final Map<String, OptionValidator> tmp = new HashMap<>(); for (final OptionValidator validator : validators) { http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java index ceecdb4..0401d58 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java @@ -55,6 +55,7 @@ public class DrillRestServer extends ResourceConfig { register(QueryResources.class); register(MetricsResources.class); register(ThreadsResources.class); + register(LogsResources.class); register(FreemarkerMvcFeature.class); register(MultiPartFeature.class); property(ServerProperties.METAINF_SERVICES_LOOKUP_DISABLE, true); http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java new file mode 100644 index 0000000..8a89d41 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java @@ -0,0 +1,211 @@ +/** + * **************************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * **************************************************************************** + */ +package org.apache.drill.exec.server.rest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Sets; +import org.apache.drill.common.exceptions.DrillRuntimeException; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.work.WorkManager; +import org.glassfish.jersey.server.mvc.Viewable; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import javax.annotation.security.RolesAllowed; +import javax.inject.Inject; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; +import javax.xml.bind.annotation.XmlRootElement; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileFilter; +import java.io.FileReader; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.Collection; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.ADMIN_ROLE; + +@Path("/") +@RolesAllowed(ADMIN_ROLE) +public class LogsResources { + + @Inject DrillRestServer.UserAuthEnabled authEnabled; + @Inject SecurityContext sc; + @Inject WorkManager work; + + private static final FileFilter file_filter = new FileFilter() { + @Override + public boolean accept(File file) { + return file.isFile(); + } + }; + private static final DateTimeFormatter format = DateTimeFormat.forPattern("MM/dd/yyyy HH:mm:ss"); + + + @GET + @Path("/logs") + @Produces(MediaType.TEXT_HTML) + public Viewable getLogs() { + Set<Log> logs = getLogsJSON(); + return ViewableWithPermissions.create(authEnabled.get(), "/rest/logs/list.ftl", sc, logs); + } + + @GET + @Path("/logs.json") + @Produces(MediaType.APPLICATION_JSON) + public Set<Log> getLogsJSON() { + Set<Log> logs = Sets.newTreeSet(); + File[] files = getLogFolder().listFiles(file_filter); + + for (File file : files) { + logs.add(new Log(file.getName(), file.length(), file.lastModified())); + } + + return logs; + } + + @GET + @Path("/log/{name}/content") + @Produces(MediaType.TEXT_HTML) + public Viewable getLog(@PathParam("name") String name) throws IOException { + LogContent content = getLogJSON(name); + return ViewableWithPermissions.create(authEnabled.get(), "/rest/logs/log.ftl", sc, content); + } + + @GET + @Path("/log/{name}/content.json") + @Produces(MediaType.APPLICATION_JSON) + public LogContent getLogJSON(@PathParam("name") final String name) throws IOException { + File file = getFileByName(getLogFolder(), name); + + final int maxLines = work.getContext().getOptionManager().getOption(ExecConstants.WEB_LOGS_MAX_LINES).num_val.intValue(); + + try (BufferedReader br = new BufferedReader(new FileReader(file))) { + Map<String, String> cache = new LinkedHashMap<String, String>(maxLines, .75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry<String, String> eldest) { + return size() > maxLines; + } + }; + + String line; + while ((line = br.readLine()) != null) { + cache.put(line, null); + } + + return new LogContent(file.getName(), cache.keySet(), maxLines); + } + } + + @GET + @Path("/log/{name}/download") + @Produces(MediaType.TEXT_PLAIN) + public Response getFullLog(@PathParam("name") final String name) { + File file = getFileByName(getLogFolder(), name); + Response.ResponseBuilder response = Response.ok(file); + response.header("Content-Disposition", String.format("attachment;filename\"%s\"", name)); + return response.build(); + } + + private File getLogFolder() { + return new File(System.getenv("DRILL_LOG_DIR")); + } + + private File getFileByName(File folder, final String name) { + File[] files = folder.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String fileName) { + return fileName.equals(name); + } + }); + if (files.length == 0) { + throw new DrillRuntimeException (name + " doesn't exist"); + } + return files[0]; + } + + + @XmlRootElement + public class Log implements Comparable<Log> { + + private String name; + private long size; + private DateTime lastModified; + + @JsonCreator + public Log (@JsonProperty("name") String name, @JsonProperty("size") long size, @JsonProperty("lastModified") long lastModified) { + this.name = name; + this.size = size; + this.lastModified = new DateTime(lastModified); + } + + public String getName() { + return name; + } + + public String getSize() { + return Math.ceil(size / 1024d) + " KB"; + } + + public String getLastModified() { + return lastModified.toString(format); + } + + @Override + public int compareTo(Log log) { + return this.getName().compareTo(log.getName()); + } + } + + @XmlRootElement + public class LogContent { + private String name; + private Collection<String> lines; + private int maxLines; + + @JsonCreator + public LogContent (@JsonProperty("name") String name, @JsonProperty("lines") Collection<String> lines, @JsonProperty("maxLines") int maxLines) { + this.name = name; + this.lines = lines; + this.maxLines = maxLines; + } + + public String getName() { + return name; + } + + public Collection<String> getLines() { return lines; } + + public int getMaxLines() { return maxLines; } + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java index b2a0fae..73019aa 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java @@ -78,6 +78,7 @@ public class ViewableWithPermissions extends Viewable { .put("showStorage", isAdmin) .put("showOptions", isAdmin) .put("showThreads", isAdmin) + .put("showLogs", isAdmin) .put("showLogin", authEnabled && showControls && !isUserLoggedIn) .put("showLogout", authEnabled && showControls && isUserLoggedIn) .put("loggedInUserName", authEnabled && showControls && http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/resources/rest/generic.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/generic.ftl b/exec/java-exec/src/main/resources/rest/generic.ftl index b3e249e..60869e7 100644 --- a/exec/java-exec/src/main/resources/rest/generic.ftl +++ b/exec/java-exec/src/main/resources/rest/generic.ftl @@ -64,6 +64,9 @@ <#if showThreads == true> <li><a href="/threads">Threads</a></li> </#if> + <#if showLogs == true> + <li><a href="/logs">Logs</a></li> + </#if> </ul> </#if> <ul class="nav navbar-nav navbar-right"> http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/resources/rest/logs/list.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/logs/list.ftl b/exec/java-exec/src/main/resources/rest/logs/list.ftl new file mode 100644 index 0000000..3d836df --- /dev/null +++ b/exec/java-exec/src/main/resources/rest/logs/list.ftl @@ -0,0 +1,55 @@ +<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + +<#include "*/generic.ftl"> +<#macro page_head> +</#macro> + +<#macro page_body> +<a href="/queries">back</a><br/> +<div class="page-header"> +</div> + +<#if (model?size > 0)> +<div class="table-responsive"> + <table class="table table-hover"> + <thead> + <td>Name</td> + <td>Size</td> + <td>Last Modified</td> + </thead> + <tbody> + <#list model as log> + <tr> + <td> + <a href="/log/${log.getName()}/content"> + <div style="height:100%;width:100%;white-space:pre-line">${log.getName()}</div> + </a> + </td> + <td> + <div style="height:100%;width:100%;white-space:pre-line">${log.getSize()}</div> + </td> + <td> + <div style="height:100%;width:100%;white-space:pre-line">${log.getLastModified()}</div> + </td> + </tr> + </#list> + </tbody> + </table> +</div> +<#else> +<div id="message" class="alert alert-info"> + <strong>No logs are available.</strong> +</div> +</#if> +</#macro> + +<@page_html/> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/resources/rest/logs/log.ftl ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/resources/rest/logs/log.ftl b/exec/java-exec/src/main/resources/rest/logs/log.ftl new file mode 100644 index 0000000..b09b57a --- /dev/null +++ b/exec/java-exec/src/main/resources/rest/logs/log.ftl @@ -0,0 +1,37 @@ +<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + +<#include "*/generic.ftl"> +<#macro page_head> +</#macro> + +<#macro page_body> +<a href="/logs">back</a><br/> +<div class="page-header"> +</div> +<h3>${model.getName()} <span class="badge alert-info">(last ${model.getMaxLines()} lines)</span></h3> +<p> + <a href="/log/${model.getName()}/download">Download Full Log</a> +</p> + <#if (model.getLines()?size > 0)> + <pre> + <#list model.getLines() as line> +${line} + </#list> + </pre> + <#else> + <div id="message" class="alert alert-info"> + <strong>Log is empty.</strong> + </div> + </#if> +</#macro> + +<@page_html/> \ No newline at end of file
