drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From par...@apache.org
Subject [5/5] drill git commit: DRILL-4571: Add link to local Drill logs from the web UI
Date Tue, 03 May 2016 17:50:48 GMT
DRILL-4571: Add link to local Drill logs from the web UI

This closes #472


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1a89a7fe
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1a89a7fe
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1a89a7fe

Branch: refs/heads/master
Commit: 1a89a7fe56533ecea8f7a7c9be6a3699925f3c96
Parents: 94b8aec
Author: Arina Ielchiieva <arina.yelchiyeva@gmail.com>
Authored: Thu Mar 31 18:43:25 2016 +0300
Committer: Parth Chandra <parthc@apache.org>
Committed: Tue May 3 10:50:09 2016 -0700

----------------------------------------------------------------------
 distribution/src/resources/logback.xml          |   2 +-
 .../org/apache/drill/exec/ExecConstants.java    |   5 +
 .../server/options/SystemOptionManager.java     |   3 +-
 .../drill/exec/server/rest/DrillRestServer.java |   1 +
 .../drill/exec/server/rest/LogsResources.java   | 211 +++++++++++++++++++
 .../server/rest/ViewableWithPermissions.java    |   1 +
 .../src/main/resources/rest/generic.ftl         |   3 +
 .../src/main/resources/rest/logs/list.ftl       |  55 +++++
 .../src/main/resources/rest/logs/log.ftl        |  37 ++++
 9 files changed, 316 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/distribution/src/resources/logback.xml
----------------------------------------------------------------------
diff --git a/distribution/src/resources/logback.xml b/distribution/src/resources/logback.xml
index 350383a..fb53dfc 100644
--- a/distribution/src/resources/logback.xml
+++ b/distribution/src/resources/logback.xml
@@ -44,7 +44,7 @@
         <maxFileSize>100MB</maxFileSize>
       </triggeringPolicy>
       <encoder>
-        <pattern>%msg</pattern>
+        <pattern>%msg%n</pattern>
       </encoder>
     </appender>
     

http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 7f216f0..17fbb7b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -303,4 +303,9 @@ public interface ExecConstants {
   StringValidator IMPERSONATION_POLICY_VALIDATOR =
       new InboundImpersonationManager.InboundImpersonationPolicyValidator(IMPERSONATION_POLICIES_KEY,
"[]");
 
+  /**
+   * Web settings
+   */
+  String WEB_LOGS_MAX_LINES = "web.logs.max_lines";
+  OptionValidator WEB_LOGS_MAX_LINES_VALIDATOR = new PositiveLongValidator(WEB_LOGS_MAX_LINES,
Integer.MAX_VALUE, 10000);
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index db78108..c35ed0e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -135,7 +135,8 @@ public class SystemOptionManager extends BaseOptionManager implements
AutoClosea
       ExecConstants.ENABLE_WINDOW_FUNCTIONS_VALIDATOR,
       ClassTransformer.SCALAR_REPLACEMENT_VALIDATOR,
       ExecConstants.ENABLE_NEW_TEXT_READER,
-      ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST
+      ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST,
+      ExecConstants.WEB_LOGS_MAX_LINES_VALIDATOR
     };
     final Map<String, OptionValidator> tmp = new HashMap<>();
     for (final OptionValidator validator : validators) {

http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
index ceecdb4..0401d58 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
@@ -55,6 +55,7 @@ public class DrillRestServer extends ResourceConfig {
     register(QueryResources.class);
     register(MetricsResources.class);
     register(ThreadsResources.class);
+    register(LogsResources.class);
     register(FreemarkerMvcFeature.class);
     register(MultiPartFeature.class);
     property(ServerProperties.METAINF_SERVICES_LOOKUP_DISABLE, true);

http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
new file mode 100644
index 0000000..8a89d41
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
@@ -0,0 +1,211 @@
+/**
+ * ****************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ****************************************************************************
+ */
+package org.apache.drill.exec.server.rest;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Sets;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.work.WorkManager;
+import org.glassfish.jersey.server.mvc.Viewable;
+import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+import javax.annotation.security.RolesAllowed;
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileReader;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.ADMIN_ROLE;
+
+@Path("/")
+@RolesAllowed(ADMIN_ROLE)
+public class LogsResources {
+
+  @Inject DrillRestServer.UserAuthEnabled authEnabled;
+  @Inject SecurityContext sc;
+  @Inject WorkManager work;
+
+  private static final FileFilter file_filter = new FileFilter() {
+    @Override
+    public boolean accept(File file) {
+      return file.isFile();
+    }
+  };
+  private static final DateTimeFormatter format = DateTimeFormat.forPattern("MM/dd/yyyy HH:mm:ss");
+
+
+  @GET
+  @Path("/logs")
+  @Produces(MediaType.TEXT_HTML)
+  public Viewable getLogs() {
+    Set<Log> logs = getLogsJSON();
+    return ViewableWithPermissions.create(authEnabled.get(), "/rest/logs/list.ftl", sc, logs);
+  }
+
+  @GET
+  @Path("/logs.json")
+  @Produces(MediaType.APPLICATION_JSON)
+  public Set<Log> getLogsJSON() {
+    Set<Log> logs = Sets.newTreeSet();
+    File[] files = getLogFolder().listFiles(file_filter);
+
+    for (File file : files) {
+      logs.add(new Log(file.getName(), file.length(), file.lastModified()));
+    }
+
+    return logs;
+  }
+
+  @GET
+  @Path("/log/{name}/content")
+  @Produces(MediaType.TEXT_HTML)
+  public Viewable getLog(@PathParam("name") String name) throws IOException {
+    LogContent content = getLogJSON(name);
+    return ViewableWithPermissions.create(authEnabled.get(), "/rest/logs/log.ftl", sc, content);
+  }
+
+  @GET
+  @Path("/log/{name}/content.json")
+  @Produces(MediaType.APPLICATION_JSON)
+  public LogContent getLogJSON(@PathParam("name") final String name) throws IOException {
+    File file = getFileByName(getLogFolder(), name);
+
+    final int maxLines = work.getContext().getOptionManager().getOption(ExecConstants.WEB_LOGS_MAX_LINES).num_val.intValue();
+
+    try (BufferedReader br = new BufferedReader(new FileReader(file))) {
+      Map<String, String> cache = new LinkedHashMap<String, String>(maxLines,
.75f, true) {
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, String> eldest) {
+          return size() > maxLines;
+        }
+      };
+
+      String line;
+      while ((line = br.readLine()) != null) {
+        cache.put(line, null);
+      }
+
+      return new LogContent(file.getName(), cache.keySet(), maxLines);
+    }
+  }
+
+  @GET
+  @Path("/log/{name}/download")
+  @Produces(MediaType.TEXT_PLAIN)
+  public Response getFullLog(@PathParam("name") final String name) {
+    File file = getFileByName(getLogFolder(), name);
+    Response.ResponseBuilder response = Response.ok(file);
+    response.header("Content-Disposition", String.format("attachment;filename\"%s\"", name));
+    return response.build();
+  }
+
+  private File getLogFolder() {
+    return new File(System.getenv("DRILL_LOG_DIR"));
+  }
+
+  private File getFileByName(File folder, final String name) {
+    File[] files = folder.listFiles(new FilenameFilter() {
+      @Override
+      public boolean accept(File dir, String fileName) {
+        return fileName.equals(name);
+      }
+    });
+    if (files.length == 0) {
+      throw new DrillRuntimeException (name + " doesn't exist");
+    }
+    return files[0];
+  }
+
+
+  @XmlRootElement
+  public class Log implements Comparable<Log> {
+
+    private String name;
+    private long size;
+    private DateTime lastModified;
+
+    @JsonCreator
+    public Log (@JsonProperty("name") String name, @JsonProperty("size") long size, @JsonProperty("lastModified")
long lastModified) {
+      this.name = name;
+      this.size = size;
+      this.lastModified = new DateTime(lastModified);
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    public String getSize() {
+      return Math.ceil(size / 1024d) + " KB";
+    }
+
+    public String getLastModified() {
+      return lastModified.toString(format);
+    }
+
+    @Override
+    public int compareTo(Log log) {
+      return this.getName().compareTo(log.getName());
+    }
+  }
+
+  @XmlRootElement
+  public class LogContent {
+    private String name;
+    private Collection<String> lines;
+    private int maxLines;
+
+    @JsonCreator
+    public LogContent (@JsonProperty("name") String name, @JsonProperty("lines") Collection<String>
lines, @JsonProperty("maxLines") int maxLines) {
+      this.name = name;
+      this.lines = lines;
+      this.maxLines = maxLines;
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    public Collection<String> getLines() { return lines; }
+
+    public int getMaxLines() { return maxLines; }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
index b2a0fae..73019aa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
@@ -78,6 +78,7 @@ public class ViewableWithPermissions extends Viewable {
         .put("showStorage", isAdmin)
         .put("showOptions", isAdmin)
         .put("showThreads", isAdmin)
+        .put("showLogs", isAdmin)
         .put("showLogin", authEnabled && showControls && !isUserLoggedIn)
         .put("showLogout", authEnabled && showControls && isUserLoggedIn)
         .put("loggedInUserName", authEnabled && showControls &&

http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/resources/rest/generic.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/generic.ftl b/exec/java-exec/src/main/resources/rest/generic.ftl
index b3e249e..60869e7 100644
--- a/exec/java-exec/src/main/resources/rest/generic.ftl
+++ b/exec/java-exec/src/main/resources/rest/generic.ftl
@@ -64,6 +64,9 @@
               <#if showThreads == true>
               <li><a href="/threads">Threads</a></li>
               </#if>
+              <#if showLogs == true>
+                  <li><a href="/logs">Logs</a></li>
+              </#if>
             </ul>
             </#if>
             <ul class="nav navbar-nav navbar-right">

http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/resources/rest/logs/list.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/logs/list.ftl b/exec/java-exec/src/main/resources/rest/logs/list.ftl
new file mode 100644
index 0000000..3d836df
--- /dev/null
+++ b/exec/java-exec/src/main/resources/rest/logs/list.ftl
@@ -0,0 +1,55 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+
+<#include "*/generic.ftl">
+<#macro page_head>
+</#macro>
+
+<#macro page_body>
+<a href="/queries">back</a><br/>
+<div class="page-header">
+</div>
+
+<#if (model?size > 0)>
+<div class="table-responsive">
+    <table class="table table-hover">
+            <thead>
+            <td>Name</td>
+            <td>Size</td>
+            <td>Last Modified</td>
+            </thead>
+        <tbody>
+            <#list model as log>
+            <tr>
+                <td>
+                    <a href="/log/${log.getName()}/content">
+                        <div style="height:100%;width:100%;white-space:pre-line">${log.getName()}</div>
+                    </a>
+                </td>
+                <td>
+                    <div style="height:100%;width:100%;white-space:pre-line">${log.getSize()}</div>
+                </td>
+                <td>
+                    <div style="height:100%;width:100%;white-space:pre-line">${log.getLastModified()}</div>
+                </td>
+            </tr>
+            </#list>
+        </tbody>
+    </table>
+</div>
+<#else>
+<div id="message" class="alert alert-info">
+    <strong>No logs are available.</strong>
+</div>
+</#if>
+</#macro>
+
+<@page_html/>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/resources/rest/logs/log.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/logs/log.ftl b/exec/java-exec/src/main/resources/rest/logs/log.ftl
new file mode 100644
index 0000000..b09b57a
--- /dev/null
+++ b/exec/java-exec/src/main/resources/rest/logs/log.ftl
@@ -0,0 +1,37 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+
+<#include "*/generic.ftl">
+<#macro page_head>
+</#macro>
+
+<#macro page_body>
+<a href="/logs">back</a><br/>
+<div class="page-header">
+</div>
+<h3>${model.getName()} <span class="badge alert-info">(last ${model.getMaxLines()}
lines)</span></h3>
+<p>
+    <a href="/log/${model.getName()}/download">Download Full Log</a>
+</p>
+    <#if (model.getLines()?size > 0)>
+    <pre>
+        <#list model.getLines() as line>
+${line}
+            </#list>
+        </pre>
+    <#else>
+    <div id="message" class="alert alert-info">
+        <strong>Log is empty.</strong>
+    </div>
+    </#if>
+</#macro>
+
+<@page_html/>
\ No newline at end of file


Mime
View raw message