KYLIN-967 kill long running query when low memory detected
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c611d7ee Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c611d7ee Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c611d7ee Branch: refs/heads/1.x-staging Commit: c611d7eefe78e8b84705b6b0e3d4af29e1686b9e Parents: bad57cb Author: shaofengshi <shaofeng...@apache.org> Authored: Fri Sep 25 16:54:43 2015 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Fri Sep 25 23:31:31 2015 +0800 ---------------------------------------------------------------------- .../kylin/rest/service/BadQueryDetector.java | 37 ++++++++++++++------ .../rest/service/BadQueryDetectorTest.java | 6 ++-- 2 files changed, 29 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c611d7ee/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java b/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java index 5f5b247..d3315c0 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java +++ b/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java @@ -20,6 +20,7 @@ package org.apache.kylin.rest.service; import java.util.ArrayList; import java.util.Collections; +import java.util.Map; import java.util.concurrent.ConcurrentMap; import org.apache.kylin.rest.request.SQLRequest; @@ -36,19 +37,21 @@ public class BadQueryDetector extends Thread { private final long detectionInterval; private final int alertMB; private final int alertRunningSec; + private final int killRunningSec; private ArrayList<Notifier> notifiers = new ArrayList<Notifier>(); public BadQueryDetector() { - this(60 * 1000, 100, 60); // 1 minute, 100 MB, 60 seconds + this(60 * 1000, 100, 60, 5 * 60); // 1 minute, 100 MB, 60 seconds, 5 minutes } - public BadQueryDetector(long detectionInterval, int alertMB, int alertRunningSec) { + public BadQueryDetector(long detectionInterval, int alertMB, int alertRunningSec, int killRunningSec) { super("BadQueryDetector"); this.setDaemon(true); this.detectionInterval = detectionInterval; this.alertMB = alertMB; this.alertRunningSec = alertRunningSec; + this.killRunningSec = killRunningSec; this.notifiers.add(new Notifier() { @Override @@ -121,15 +124,6 @@ public class BadQueryDetector extends Thread { ArrayList<Entry> entries = new ArrayList<Entry>(runningQueries.values()); Collections.sort(entries); - // report if low memory - if (getSystemAvailMB() < alertMB) { - logger.info("System free memory less than " + alertMB + " MB. " + entries.size() + " queries running."); - for (int i = 0; i < entries.size(); i++) { - Entry e = entries.get(i); - notify("Low mem", (int) ((now - e.startTime) / 1000), e.sqlRequest.getSql()); - } - } - // report if query running long for (Entry e : entries) { int runningSec = (int) ((now - e.startTime) / 1000); @@ -139,6 +133,27 @@ public class BadQueryDetector extends Thread { break; // entries are sorted by startTime } } + + // report if low memory + if (getSystemAvailMB() < alertMB) { + logger.info("System free memory less than " + alertMB + " MB. " + entries.size() + " queries running."); + + for (Map.Entry<Thread, Entry> mapEntry : runningQueries.entrySet()) { + Entry e = mapEntry.getValue(); + int duration = (int) ((now - e.startTime) / 1000); + if (duration > killRunningSec) { + notify("Kill", duration, e.sqlRequest.getSql()); + Thread queryThread = mapEntry.getKey(); + killQueryThread(queryThread); + } else { + notify("Low mem", duration, e.sqlRequest.getSql()); + } + } + } + } + + private void killQueryThread(Thread thread) { + thread.interrupt(); } public static final int ONE_MB = 1024 * 1024; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c611d7ee/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java index c849efd..d7e8ba9 100644 --- a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java +++ b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java @@ -35,7 +35,7 @@ public class BadQueryDetectorTest { String mockSql = "select * from just_a_test"; final ArrayList<String[]> alerts = new ArrayList<>(); - BadQueryDetector badQueryDetector = new BadQueryDetector(alertRunningSec * 1000, alertMB, alertRunningSec); + BadQueryDetector badQueryDetector = new BadQueryDetector(alertRunningSec * 1000, alertMB, alertRunningSec, alertRunningSec * 5); badQueryDetector.registerNotifier(new BadQueryDetector.Notifier() { @Override public void badQueryFound(String adj, int runningSec, String sql) { @@ -63,7 +63,7 @@ public class BadQueryDetectorTest { // first check founds Low mem assertArrayEquals(new String[] { "Low mem", mockSql }, alerts.get(0)); // second check founds Low mem & Slow - assertArrayEquals(new String[] { "Low mem", mockSql }, alerts.get(1)); - assertArrayEquals(new String[] { "Slow", mockSql }, alerts.get(2)); + assertArrayEquals(new String[] { "Slow", mockSql }, alerts.get(1)); + assertArrayEquals(new String[] { "Low mem", mockSql }, alerts.get(2)); } }