kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lid...@apache.org
Subject kylin git commit: KYLIN-2880 Enhance BadQueryDetector to include query id
Date Tue, 30 Jan 2018 13:52:09 GMT
Repository: kylin
Updated Branches:
  refs/heads/master c704f7cda -> 6a2ee80c9


KYLIN-2880 Enhance BadQueryDetector to include query id

Signed-off-by: lidongsjtu <lidong@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6a2ee80c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6a2ee80c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6a2ee80c

Branch: refs/heads/master
Commit: 6a2ee80c9ec19be1d7e72ed8b587fbc48a195867
Parents: c704f7c
Author: Zhong <nju_yaho@apache.org>
Authored: Tue Jan 30 14:13:18 2018 +0800
Committer: lidongsjtu <lidong@apache.org>
Committed: Tue Jan 30 21:45:18 2018 +0800

----------------------------------------------------------------------
 .../kylin/metadata/badquery/BadQueryEntry.java  | 14 +++++++++-
 .../badquery/BadQueryHistoryManagerTest.java    | 15 ++++++----
 .../kylin/rest/service/BadQueryDetector.java    | 29 +++++++++++---------
 .../apache/kylin/rest/service/QueryService.java |  6 ++--
 .../rest/service/BadQueryDetectorTest.java      |  6 ++--
 .../rest/service/KylinQueryTimeoutTest.java     |  4 ++-
 6 files changed, 49 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
index 1e9311f..56b27ca 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
@@ -47,8 +47,11 @@ public class BadQueryEntry extends RootPersistentEntity implements Comparable<Ba
     private String thread;
     @JsonProperty("user")
     private String user;
+    @JsonProperty("query_id")
+    private String queryId;
 
-    public BadQueryEntry(String sql, String adj, long startTime, float runningSec, String
server, String thread, String user) {
+    public BadQueryEntry(String sql, String adj, long startTime, float runningSec, String
server, String thread,
+            String user, String queryId) {
         this.updateRandomUuid();
         this.adj = adj;
         this.sql = sql;
@@ -57,12 +60,21 @@ public class BadQueryEntry extends RootPersistentEntity implements Comparable<Ba
         this.server = server;
         this.thread = thread;
         this.user = user;
+        this.queryId = queryId;
     }
 
     public BadQueryEntry() {
 
     }
 
+    public String getQueryId() {
+        return queryId;
+    }
+
+    public void setQueryId(String queryId) {
+        this.queryId = queryId;
+    }
+
     public String getUser() {
         return user;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
index c4a8229..5001cd2 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.util.NavigableSet;
+import java.util.UUID;
 
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.JsonUtil;
@@ -65,7 +66,8 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase
{
     public void testAddEntryToProject() throws IOException {
         KylinConfig kylinConfig = getTestConfig();
         BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig);
-        BadQueryEntry entry = new BadQueryEntry("sql", "adj", 1459362239992L, 100, "server",
"t-0", "user");
+        BadQueryEntry entry = new BadQueryEntry("sql", "adj", 1459362239992L, 100, "server",
"t-0", "user",
+                UUID.randomUUID().toString());
         BadQueryHistory history = manager.upsertEntryToProject(entry, "default");
         NavigableSet<BadQueryEntry> entries = history.getEntries();
         assertEquals(4, entries.size());
@@ -81,7 +83,8 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase
{
         assertEquals("t-0", newEntry.getThread());
 
         for (int i = 0; i < kylinConfig.getBadQueryHistoryNum(); i++) {
-            BadQueryEntry tmp = new BadQueryEntry("sql", "adj", 1459362239993L + i, 100 +
i, "server", "t-0", "user");
+            BadQueryEntry tmp = new BadQueryEntry("sql", "adj", 1459362239993L + i, 100 +
i, "server", "t-0", "user",
+                    UUID.randomUUID().toString());
             history = manager.upsertEntryToProject(tmp, "default");
         }
         assertEquals(kylinConfig.getBadQueryHistoryNum(), history.getEntries().size());
@@ -92,14 +95,16 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase
{
         KylinConfig kylinConfig = getTestConfig();
         BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig);
 
-        manager.upsertEntryToProject(new BadQueryEntry("sql", "adj", 1459362239000L, 100,
"server", "t-0", "user"),
+        String queryId = UUID.randomUUID().toString();
+        manager.upsertEntryToProject(
+                new BadQueryEntry("sql", "adj", 1459362239000L, 100, "server", "t-0", "user",
queryId),
                 "default");
         BadQueryHistory history = manager.upsertEntryToProject(
-                new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user"),
"default");
+                new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user",
queryId), "default");
 
         NavigableSet<BadQueryEntry> entries = history.getEntries();
         BadQueryEntry newEntry = entries
-                .floor(new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1",
"user"));
+                .floor(new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1",
"user", queryId));
         System.out.println(newEntry);
         assertEquals("adj2", newEntry.getAdj());
         assertEquals("server2", newEntry.getServer());

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
index 4f7bccf..51f49a7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
@@ -102,15 +102,15 @@ public class BadQueryDetector extends Thread {
         for (Notifier notifier : notifiers) {
             try {
                 notifier.badQueryFound(adj, runningSec, //
-                        e.startTime, e.sqlRequest.getProject(), e.sqlRequest.getSql(), e.user,
e.thread);
+                        e.startTime, e.sqlRequest.getProject(), e.sqlRequest.getSql(), e.user,
e.thread, e.queryId);
             } catch (Exception ex) {
                 logger.error("", ex);
             }
         }
     }
 
-    public void queryStart(Thread thread, SQLRequest sqlRequest, String user) {
-        runningQueries.put(thread, new Entry(sqlRequest, user, thread));
+    public void queryStart(Thread thread, SQLRequest sqlRequest, String user, String queryId)
{
+        runningQueries.put(thread, new Entry(sqlRequest, user, thread, queryId));
     }
 
     public void queryEnd(Thread thread) {
@@ -155,7 +155,7 @@ public class BadQueryDetector extends Thread {
 
             if (runningSec >= alertRunningSec) {
                 notify(BadQueryEntry.ADJ_SLOW, e);
-                dumpStackTrace(e.thread);
+                dumpStackTrace(e.thread, e.queryId);
             } else {
                 break; // entries are sorted by startTime
             }
@@ -175,12 +175,13 @@ public class BadQueryDetector extends Thread {
     }
 
     // log the stack trace of bad query thread for further analysis
-    private void dumpStackTrace(Thread t) {
+    private void dumpStackTrace(Thread t, String queryId) {
         int maxStackTraceDepth = kylinConfig.getBadQueryStackTraceDepth();
         int current = 0;
 
         StackTraceElement[] stackTrace = t.getStackTrace();
-        StringBuilder buf = new StringBuilder("Problematic thread 0x" + Long.toHexString(t.getId()));
+        StringBuilder buf = new StringBuilder(
+                "Problematic thread 0x" + Long.toHexString(t.getId()) + " " + t.getName()
+ ", query id: " + queryId);
         buf.append("\n");
         for (StackTraceElement e : stackTrace) {
             if (++current > maxStackTraceDepth) {
@@ -193,15 +194,15 @@ public class BadQueryDetector extends Thread {
 
     public interface Notifier {
         void badQueryFound(String adj, float runningSec, long startTime, String project,
String sql, String user,
-                Thread t);
+                Thread t, String queryId);
     }
 
     private class LoggerNotifier implements Notifier {
         @Override
         public void badQueryFound(String adj, float runningSec, long startTime, String project,
String sql, String user,
-                Thread t) {
-            logger.info("{} query has been running {} seconds (project:{}, thread: 0x{},
user:{}) -- {}", adj,
-                    runningSec, project, Long.toHexString(t.getId()), user, sql);
+                Thread t, String queryId) {
+            logger.info("{} query has been running {} seconds (project:{}, thread: 0x{},
user:{}, query id:{}) -- {}",
+                    adj, runningSec, project, Long.toHexString(t.getId()), user, queryId,
sql);
         }
     }
 
@@ -220,10 +221,10 @@ public class BadQueryDetector extends Thread {
 
         @Override
         public void badQueryFound(String adj, float runningSec, long startTime, String project,
String sql, String user,
-                Thread t) {
+                Thread t, String queryId) {
             try {
                 BadQueryEntry entry = new BadQueryEntry(sql, adj, startTime, runningSec,
serverHostname, t.getName(),
-                        user);
+                        user, queryId);
                 badQueryManager.upsertEntryToProject(entry, project);
             } catch (IOException e) {
                 logger.error("Error in bad query persistence.", e);
@@ -236,12 +237,14 @@ public class BadQueryDetector extends Thread {
         final long startTime;
         final Thread thread;
         final String user;
+        final String queryId;
 
-        Entry(SQLRequest sqlRequest, String user, Thread thread) {
+        Entry(SQLRequest sqlRequest, String user, Thread thread, String queryId) {
             this.sqlRequest = sqlRequest;
             this.startTime = System.currentTimeMillis();
             this.thread = thread;
             this.user = user;
+            this.queryId = queryId;
         }
 
         @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 56fab34..841cad9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -182,11 +182,11 @@ public class QueryService extends BasicService {
         return getMetadata(getCubeManager(), project);
     }
 
-    public SQLResponse query(SQLRequest sqlRequest) throws Exception {
+    public SQLResponse query(SQLRequest sqlRequest, String queryId) throws Exception {
         SQLResponse ret = null;
         try {
             final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-            badQueryDetector.queryStart(Thread.currentThread(), sqlRequest, user);
+            badQueryDetector.queryStart(Thread.currentThread(), sqlRequest, user, queryId);
 
             ret = queryWithSqlMassage(sqlRequest);
             return ret;
@@ -492,7 +492,7 @@ public class QueryService extends BasicService {
         try {
             final boolean isSelect = QueryUtil.isSelectStatement(sqlRequest.getSql());
             if (isSelect) {
-                sqlResponse = query(sqlRequest);
+                sqlResponse = query(sqlRequest, queryContext.getQueryId());
                 Trace.addTimelineAnnotation("query almost done");
             } else if (kylinConfig.isPushDownEnabled() && kylinConfig.isPushDownUpdateEnabled())
{
                 sqlResponse = update(sqlRequest);

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
index d61dfbe..61e52b7 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 
 import java.util.ArrayList;
+import java.util.UUID;
 
 import org.apache.kylin.common.util.LocalFileMetadataTestCase;
 import org.apache.kylin.metadata.badquery.BadQueryEntry;
@@ -53,7 +54,8 @@ public class BadQueryDetectorTest extends LocalFileMetadataTestCase {
         BadQueryDetector badQueryDetector = new BadQueryDetector(alertRunningSec * 1000,
alertMB, alertRunningSec, 1000);
         badQueryDetector.registerNotifier(new BadQueryDetector.Notifier() {
             @Override
-            public void badQueryFound(String adj, float runningSec, long startTime, String
project, String sql, String user, Thread t) {
+            public void badQueryFound(String adj, float runningSec, long startTime, String
project, String sql,
+                    String user, Thread t, String queryId) {
                 alerts.add(new String[] { adj, sql });
             }
         });
@@ -64,7 +66,7 @@ public class BadQueryDetectorTest extends LocalFileMetadataTestCase {
 
             SQLRequest sqlRequest = new SQLRequest();
             sqlRequest.setSql(mockSql);
-            badQueryDetector.queryStart(Thread.currentThread(), sqlRequest, "user");
+            badQueryDetector.queryStart(Thread.currentThread(), sqlRequest, "user", UUID.randomUUID().toString());
 
             // make sure bad query check happens twice
             Thread.sleep((alertRunningSec * 2 + 1) * 1000);

http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
b/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
index 6a61b2f..38b3270 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
@@ -17,7 +17,9 @@
  */
 
 package org.apache.kylin.rest.service;
+
 import java.sql.SQLException;
+import java.util.UUID;
 
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.exceptions.KylinTimeoutException;
@@ -69,7 +71,7 @@ public class KylinQueryTimeoutTest extends LocalFileMetadataTestCase {
         SQLRequest request = new SQLRequest();
         request.setProject("default");
         request.setSql("select count(*) from STREAMING_TABLE");
-        detector.queryStart(Thread.currentThread(), request, "ADMIN");
+        detector.queryStart(Thread.currentThread(), request, "ADMIN", UUID.randomUUID().toString());
         try {
             QueryACLTestUtil.mockQuery("default", "select * from STREAMING_TABLE");
         } finally{


Mime
View raw message