This is an automated email from the ASF dual-hosted git repository.
dspavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new e652a5dd Ignite-28630: additional connection issues retries and
hangover (#207)
e652a5dd is described below
commit e652a5dd4c22beb927181c706889f0ff1068d2d9
Author: ignitetcbot <[email protected]>
AuthorDate: Thu May 7 18:31:04 2026 +0300
Ignite-28630: additional connection issues retries and hangover (#207)
IGNITE-28630: improve retries, diagnostics and UI log analysis for TCBot
---------
Co-authored-by: Dmitriy Pavlov <[email protected]>
---
.../org/apache/ignite/ci/web/model/Version.java | 2 +-
.../{TaskResult.java => AppLogEntry.java} | 31 ++-
.../{TaskResult.java => AppLogSummaryLink.java} | 11 +-
.../ci/web/rest/monitoring/MonitoringService.java | 299 +++++++++++++++++++++
.../ignite/ci/web/rest/monitoring/TaskResult.java | 2 +
.../src/main/webapp/buildtime.html | 4 +-
ignite-tc-helper-web/src/main/webapp/index.html | 4 +-
ignite-tc-helper-web/src/main/webapp/index0.html | 4 +-
.../src/main/webapp/monitoring-log.html | 136 ++++++++++
.../src/main/webapp/monitoring.html | 58 +++-
.../exeption/ServiceUnavailableException.java | 83 ++++++
.../interceptor/MonitoredTaskInterceptor.java | 30 ++-
.../apache/ignite/tcbot/common/util/HttpUtil.java | 49 +++-
.../githubignited/GitHubConnIgnitedImpl.java | 141 ++++++++--
.../ignite/githubservice/GitHubConnectionImpl.java | 120 +++++++--
.../ignite/tcignited/buildref/BuildRefSync.java | 135 +++++-----
.../tcservice/TeamcityServiceConnection.java | 125 ++++++++-
.../tcservice/http/ITeamcityHttpConnection.java | 2 +
18 files changed, 1088 insertions(+), 148 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
index a338f177..36f21dac 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java
@@ -28,7 +28,7 @@ package org.apache.ignite.ci.web.model;
public static final String GITHUB_REF =
"https://github.com/apache/ignite-teamcity-bot";;
/** TC Bot Version. */
- public static final String VERSION = "20220422";
+ public static final String VERSION = "20260429";
/** Java version, where Web App is running. */
public String javaVer;
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/TaskResult.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/AppLogEntry.java
similarity index 60%
copy from
ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/TaskResult.java
copy to
ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/AppLogEntry.java
index 9003f43b..1254bcba 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/TaskResult.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/AppLogEntry.java
@@ -16,11 +16,30 @@
*/
package org.apache.ignite.ci.web.rest.monitoring;
+/** Application log entry. */
@SuppressWarnings("WeakerAccess")
-public class TaskResult {
- public String name;
- public String start;
- public Integer count;
- public String end;
- public String result;
+public class AppLogEntry {
+ /** Timestamp. */
+ public String timestamp;
+
+ /** Level. */
+ public String level;
+
+ /** Source file. */
+ public String file;
+
+ /** Full log text, including stacktrace continuation lines. */
+ public String text;
+
+ /** Short log summary. */
+ public String summary;
+
+ /** Service URL detected in log text. */
+ public String serviceUrl;
+
+ /** Service host detected in log text. */
+ public String serviceHost;
+
+ /** HTTP response code detected in log text. */
+ public Integer responseCode;
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/TaskResult.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/AppLogSummaryLink.java
similarity index 85%
copy from
ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/TaskResult.java
copy to
ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/AppLogSummaryLink.java
index 9003f43b..54807407 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/TaskResult.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/AppLogSummaryLink.java
@@ -16,11 +16,12 @@
*/
package org.apache.ignite.ci.web.rest.monitoring;
+/** App log summary link parameters. */
@SuppressWarnings("WeakerAccess")
-public class TaskResult {
+public class AppLogSummaryLink {
+ /** Start timestamp. */
+ public Long startTs;
+
+ /** Link title. */
public String name;
- public String start;
- public Integer count;
- public String end;
- public String result;
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
index 58fbfceb..c0d1c9b7 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
@@ -17,6 +17,17 @@
package org.apache.ignite.ci.web.rest.monitoring;
import com.google.common.base.Strings;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheMetrics;
@@ -25,6 +36,7 @@ import org.apache.ignite.ci.web.CtxListener;
import org.apache.ignite.ci.web.model.SimpleResult;
import org.apache.ignite.tcbot.common.interceptor.AutoProfilingInterceptor;
import org.apache.ignite.tcbot.common.interceptor.MonitoredTaskInterceptor;
+import org.apache.ignite.tcbot.common.conf.TcBotWorkDir;
import org.apache.ignite.tcbot.engine.conf.INotificationChannel;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.conf.NotificationsConfig;
@@ -38,15 +50,51 @@ import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
@Path("monitoring")
@Produces(MediaType.APPLICATION_JSON)
public class MonitoringService {
+ /** Log line start. */
+ private static final Pattern LOG_ENTRY_START = Pattern.compile(
+ "^(\\d{4}-\\d{2}-\\d{2}
\\d{2}:\\d{2}:\\d{2}\\.\\d{3})\\s+(\\S+)\\s+.*");
+
+ /** Service URL in log text. */
+ private static final Pattern SERVICE_URL = Pattern.compile("(?:Service
|Response URL: |url=)(https?://[^\\s,\\]]+)");
+
+ /** Service host in log text. */
+ private static final Pattern SERVICE_HOST =
Pattern.compile("(?:host=|Response host: )([^\\s,\\]\\)]+)");
+
+ /** HTTP response code in log text. */
+ private static final Pattern RESPONSE_CODE = Pattern.compile(
+ "(?:Invalid Response Code|Service Unavailable Response
Code)\\s*:\\s*(\\d{3})|HTTP\\s+(\\d{3})|Response:\\s*(\\d{3})");
+
+ /** Exception summary in log text. */
+ private static final Pattern EXCEPTION_SUMMARY = Pattern.compile(
+ "(?m)^(?:Caused by: )?([\\w.$]+(?:Exception|Error): .+)$");
+
+ /** Secret-like values in log text. */
+ private static final Pattern SECRET_VALUE = Pattern.compile(
+
"(?i)(authorization:\\s*(?:basic|bearer|token)\\s+|(?:access_token|auth_token|token|password|passwd|pwd|secret)=)"
+
+ "([^\\s&\"'<>]+)");
+
+ /** JSON secret-like values in log text. */
+ private static final Pattern JSON_SECRET_VALUE = Pattern.compile(
+
"(?i)(\"(?:access_token|auth_token|token|password|passwd|pwd|secret)\"\\s*:\\s*\")([^\"]+)(\")");
+
+ /** Max summary length. */
+ private static final int SUMMARY_LIMIT = 240;
+
+ /** Log timestamp format. */
+ private static final DateTimeFormatter LOG_TS_FORMAT =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
+
/** Context. */
@Context
private ServletContext ctx;
@@ -63,13 +111,264 @@ public class MonitoringService {
final TaskResult res = new TaskResult();
res.name = invocation.name();
res.start = invocation.start();
+ res.startTs = invocation.startTs();
res.end = invocation.end();
+ res.endTs = invocation.endTs();
res.result = invocation.result();
res.count = invocation.count();
return res;
}).collect(Collectors.toList());
}
+ @GET
+ @Path("appLogSummaryLink")
+ public AppLogSummaryLink getAppLogSummaryLink() {
+ MonitoredTaskInterceptor instance =
CtxListener.getInjector(ctx).getInstance(MonitoredTaskInterceptor.class);
+
+ AppLogSummaryLink res = new AppLogSummaryLink();
+ res.startTs = instance.startedTs();
+ res.name = "Application log since startup";
+
+ return res;
+ }
+
+ @GET
+ @Path("taskLog")
+ public List<AppLogEntry> getTaskLog(@QueryParam("startTs") long startTs,
@QueryParam("endTs") long endTs) {
+ if (startTs <= 0)
+ return new ArrayList<>();
+
+ long actualEndTs = endTs > 0 ? endTs : System.currentTimeMillis();
+
+ if (actualEndTs < startTs)
+ actualEndTs = startTs;
+
+ File[] files = appLogFiles(startTs);
+
+ List<AppLogEntry> res = new ArrayList<>();
+
+ for (File file : files)
+ readLogEntries(file, startTs, actualEndTs, res);
+
+ return res;
+ }
+
+ /**
+ * @param startTs Task start timestamp.
+ */
+ private File[] appLogFiles(long startTs) {
+ File tcbotLogs = new File(TcBotWorkDir.resolveWorkDir(), "tcbot_logs");
+
+ File[] files = tcbotLogs.listFiles(file -> file.isFile()
+ && file.getName().endsWith(".log")
+ && !file.getName().startsWith("monitoring")
+ && file.lastModified() >= startTs - 60 * 60 * 1000L);
+
+ if (files == null)
+ return new File[0];
+
+ Arrays.sort(files, Comparator.comparingLong(File::lastModified));
+
+ return files;
+ }
+
+ /**
+ * @param file Log file.
+ * @param startTs Start timestamp.
+ * @param endTs End timestamp.
+ * @param res Result.
+ */
+ private void readLogEntries(File file, long startTs, long endTs,
List<AppLogEntry> res) {
+ try (BufferedReader reader = Files.newBufferedReader(file.toPath(),
StandardCharsets.UTF_8)) {
+ AppLogEntry cur = null;
+ boolean collect = false;
+
+ for (String line = reader.readLine(); line != null; line =
reader.readLine()) {
+ Matcher matcher = LOG_ENTRY_START.matcher(line);
+
+ if (matcher.matches()) {
+ enrichLogEntry(cur);
+
+ cur = null;
+ collect = false;
+
+ long ts = logTimestamp(matcher.group(1));
+ String level = matcher.group(2);
+
+ if (ts >= startTs && ts <= endTs &&
isWarningOrError(level)) {
+ cur = new AppLogEntry();
+ cur.timestamp = matcher.group(1);
+ cur.level = level;
+ cur.file = file.getName();
+ cur.text = line;
+
+ res.add(cur);
+ collect = true;
+ }
+ }
+ else if (collect)
+ cur.text += System.lineSeparator() + line;
+ }
+
+ enrichLogEntry(cur);
+ }
+ catch (IOException ignored) {
+ // Monitoring page must remain available even if a log file is
being rotated.
+ }
+ }
+
+ /**
+ * @param entry Log entry.
+ */
+ private void enrichLogEntry(AppLogEntry entry) {
+ if (entry == null || entry.text == null)
+ return;
+
+ entry.text = sanitizeLogText(entry.text);
+ entry.serviceUrl = serviceUrl(entry.text);
+ entry.serviceHost = serviceHost(entry.text, entry.serviceUrl);
+ entry.responseCode = responseCode(entry.text);
+ entry.summary = summary(entry);
+ }
+
+ /**
+ * @param text Log text.
+ */
+ private String sanitizeLogText(String text) {
+ String sanitized =
SECRET_VALUE.matcher(text).replaceAll("$1<redacted>");
+
+ return
JSON_SECRET_VALUE.matcher(sanitized).replaceAll("$1<redacted>$3");
+ }
+
+ /**
+ * @param text Log text.
+ */
+ private String serviceUrl(String text) {
+ Matcher matcher = SERVICE_URL.matcher(text);
+
+ if (!matcher.find())
+ return null;
+
+ return trimUrl(matcher.group(1));
+ }
+
+ /**
+ * @param text Log text.
+ * @param serviceUrl Service URL.
+ */
+ private String serviceHost(String text, String serviceUrl) {
+ if (!Strings.isNullOrEmpty(serviceUrl)) {
+ try {
+ return new URL(serviceUrl).getHost();
+ }
+ catch (MalformedURLException ignored) {
+ // Try explicit host fields below.
+ }
+ }
+
+ Matcher matcher = SERVICE_HOST.matcher(text);
+
+ return matcher.find() ? matcher.group(1) : null;
+ }
+
+ /**
+ * @param text Log text.
+ */
+ private Integer responseCode(String text) {
+ Matcher matcher = RESPONSE_CODE.matcher(text);
+
+ if (!matcher.find())
+ return null;
+
+ for (int i = 1; i <= matcher.groupCount(); i++) {
+ if (matcher.group(i) != null)
+ return Integer.valueOf(matcher.group(i));
+ }
+
+ return null;
+ }
+
+ /**
+ * @param entry Log entry.
+ */
+ private String summary(AppLogEntry entry) {
+ String msg = exceptionSummary(entry.text);
+
+ if (Strings.isNullOrEmpty(msg))
+ msg = firstLine(entry.text);
+
+ if (entry.responseCode != null &&
!Strings.isNullOrEmpty(entry.serviceHost))
+ msg = "HTTP " + entry.responseCode + " from " + entry.serviceHost
+ ": " + msg;
+ else if (!Strings.isNullOrEmpty(entry.serviceHost))
+ msg = "Host " + entry.serviceHost + ": " + msg;
+
+ return limit(msg);
+ }
+
+ /**
+ * @param text Log text.
+ */
+ private String exceptionSummary(String text) {
+ Matcher matcher = EXCEPTION_SUMMARY.matcher(text);
+ String res = null;
+
+ while (matcher.find())
+ res = matcher.group(1);
+
+ return res;
+ }
+
+ /**
+ * @param text Text.
+ */
+ private String firstLine(String text) {
+ int end = text.indexOf(System.lineSeparator());
+
+ return end >= 0 ? text.substring(0, end) : text;
+ }
+
+ /**
+ * @param url URL.
+ */
+ private String trimUrl(String url) {
+ while (url.endsWith(":") || url.endsWith(".") || url.endsWith(";"))
+ url = url.substring(0, url.length() - 1);
+
+ return url;
+ }
+
+ /**
+ * @param text Text.
+ */
+ private String limit(String text) {
+ if (text == null || text.length() <= SUMMARY_LIMIT)
+ return text;
+
+ return text.substring(0, SUMMARY_LIMIT - 3) + "...";
+ }
+
+ /**
+ * @param timestamp Timestamp.
+ */
+ private long logTimestamp(String timestamp) {
+ try {
+ return LocalDateTime.parse(timestamp, LOG_TS_FORMAT)
+ .atZone(ZoneId.systemDefault())
+ .toInstant()
+ .toEpochMilli();
+ }
+ catch (DateTimeParseException e) {
+ return 0;
+ }
+ }
+
+ /**
+ * @param level Log level.
+ */
+ private boolean isWarningOrError(String level) {
+ return "WARN".equals(level) || "ERROR".equals(level);
+ }
+
@GET
@PermitAll
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/TaskResult.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/TaskResult.java
index 9003f43b..3e15479e 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/TaskResult.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/TaskResult.java
@@ -20,7 +20,9 @@ package org.apache.ignite.ci.web.rest.monitoring;
public class TaskResult {
public String name;
public String start;
+ public Long startTs;
public Integer count;
public String end;
+ public Long endTs;
public String result;
}
diff --git a/ignite-tc-helper-web/src/main/webapp/buildtime.html
b/ignite-tc-helper-web/src/main/webapp/buildtime.html
index 7d1a8776..9dff77b2 100644
--- a/ignite-tc-helper-web/src/main/webapp/buildtime.html
+++ b/ignite-tc-helper-web/src/main/webapp/buildtime.html
@@ -65,7 +65,7 @@ function loadAnalytics() {
function loadData() {
$("#loadStatus").html("⌛ Please wait");
- $("#version").html(" " + "<a href=\"monitoring.html\">TC Bot Moniroting
Page</a> <br>");
+ $("#version").html(" " + "<a href=\"monitoring.html\">TC Bot Monitoring
Page</a> <br>");
$.ajax({
url: "rest/branches/version",
success: showVersionInfo,
@@ -135,4 +135,4 @@ function loadData() {
<div id="version"></div>
</body>
-</html>
\ No newline at end of file
+</html>
diff --git a/ignite-tc-helper-web/src/main/webapp/index.html
b/ignite-tc-helper-web/src/main/webapp/index.html
index 894f8915..6b92ed06 100644
--- a/ignite-tc-helper-web/src/main/webapp/index.html
+++ b/ignite-tc-helper-web/src/main/webapp/index.html
@@ -22,7 +22,7 @@ $(document).ready(function() {
function loadData() {
$("#loadStatus").html("⌛ Please wait");
- $("#version").html(" " + "<a href=\"monitoring.html\">TC Bot Moniroting
Page</a> <br>");
+ $("#version").html(" " + "<a href=\"monitoring.html\">TC Bot Monitoring
Page</a> <br>");
$.ajax({
url: "rest/branches/version",
success: showVersionInfo,
@@ -61,4 +61,4 @@ function loadData() {
<div id="version"></div>
</body>
-</html>
\ No newline at end of file
+</html>
diff --git a/ignite-tc-helper-web/src/main/webapp/index0.html
b/ignite-tc-helper-web/src/main/webapp/index0.html
index 1ff1f0a7..3daee075 100644
--- a/ignite-tc-helper-web/src/main/webapp/index0.html
+++ b/ignite-tc-helper-web/src/main/webapp/index0.html
@@ -157,9 +157,9 @@ Statistics: <br>
Other: <br>
<a href="ignval.html">Ignite Log Values pretty-print</a>
-<a href="monitoring.html">Bot moniroting page</a> <br>
+<a href="monitoring.html">Bot monitoring page</a> <br>
<div id="loadStatus"></div>
<div id="version"></div>
</body>
-</html>
\ No newline at end of file
+</html>
diff --git a/ignite-tc-helper-web/src/main/webapp/monitoring-log.html
b/ignite-tc-helper-web/src/main/webapp/monitoring-log.html
new file mode 100644
index 00000000..cc32ce17
--- /dev/null
+++ b/ignite-tc-helper-web/src/main/webapp/monitoring-log.html
@@ -0,0 +1,136 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <title>Apache Ignite Teamcity Bot - Application log summary</title>
+
+ <link rel="icon" href="img/leaf-icon-png-7066.png">
+ <link rel="stylesheet" href="css/style-1.5.css">
+
+ <script src="https://code.jquery.com/jquery-1.12.4.js";></script>
+ <script src="js/common-1.6.js"></script>
+</head>
+<body>
+<script>
+ $(document).ready(function() {
+ loadLogSummary();
+ });
+
+ function queryParam(name) {
+ var params = new URLSearchParams(window.location.search);
+
+ return params.get(name);
+ }
+
+ function escapeHtml(str) {
+ return String(str || "")
+ .replace(/&/g, "&")
+ .replace(/</g, "<")
+ .replace(/>/g, ">")
+ .replace(/"/g, """)
+ .replace(/'/g, "'");
+ }
+
+ function loadLogSummary() {
+ var startTs = queryParam("startTs");
+ var endTs = queryParam("endTs") || "0";
+ var name = queryParam("name") || "";
+
+ $("#taskName").text(name);
+ $("#loadStatus").html("⌛ Please wait");
+
+ $.ajax({
+ url: "rest/monitoring/taskLog",
+ data: {
+ startTs: startTs,
+ endTs: endTs
+ },
+ success: function(result) {
+ $("#loadStatus").html("");
+ showLogSummary(result);
+ },
+ error: showErrInLoadStatus
+ });
+ }
+
+ function showLogSummary(result) {
+ $("#summary").text("Warnings/errors found: " + result.length);
+
+ if (result.length === 0) {
+ $("#logEntries").html("<p>No WARN/ERROR records found for the
selected interval.</p>");
+
+ return;
+ }
+
+ var res = "";
+
+ for (var i = 0; i < result.length; i++) {
+ var entry = result[i];
+ var summary = entry.summary || firstLine(entry.text);
+
+ res += "<h3>" + escapeHtml(entry.timestamp) + " " +
escapeHtml(entry.level) +
+ " [" + escapeHtml(entry.file) + "]</h3>";
+ res += "<div class='log-meta'>" + logMeta(entry) + "</div>";
+ res += "<details>";
+ res += "<summary>" + escapeHtml(summary) + "</summary>";
+ res += "<pre>" + escapeHtml(entry.text) + "</pre>";
+ res += "</details>";
+ }
+
+ $("#logEntries").html(res);
+ }
+
+ function logMeta(entry) {
+ var meta = [];
+
+ if (entry.responseCode)
+ meta.push("HTTP " + escapeHtml(entry.responseCode));
+
+ if (entry.serviceHost)
+ meta.push("Host: " + escapeHtml(entry.serviceHost));
+
+ if (entry.serviceUrl)
+ meta.push("<a target='_blank' rel='noopener noreferrer' href='" +
escapeHtml(entry.serviceUrl) + "'>" +
+ escapeHtml(entry.serviceUrl) + "</a>");
+
+ return meta.join(" | ");
+ }
+
+ function firstLine(text) {
+ var lines = String(text || "").split(/\r?\n/);
+
+ return lines.length > 0 ? lines[0] : "";
+ }
+</script>
+<style>
+ details {
+ margin-bottom: 16px;
+ }
+
+ summary {
+ cursor: pointer;
+ font-weight: bold;
+ overflow-wrap: anywhere;
+ }
+
+ pre {
+ white-space: pre-wrap;
+ overflow-wrap: anywhere;
+ }
+
+ .log-meta {
+ color: #555;
+ margin: 0 0 4px 0;
+ overflow-wrap: anywhere;
+ }
+</style>
+
+<h2>Application Log Warnings/Errors</h2>
+<b>Task:</b> <span id="taskName"></span>
+<br>
+<b id="summary"></b>
+<div id="loadStatus"></div>
+<div id="logEntries" style="font-family: monospace"></div>
+
+</body>
+</html>
diff --git a/ignite-tc-helper-web/src/main/webapp/monitoring.html
b/ignite-tc-helper-web/src/main/webapp/monitoring.html
index aceec3ef..0cf0703c 100644
--- a/ignite-tc-helper-web/src/main/webapp/monitoring.html
+++ b/ignite-tc-helper-web/src/main/webapp/monitoring.html
@@ -2,7 +2,7 @@
<html lang="en">
<head>
<meta charset="UTF-8">
- <title>Apache Ignite Teamcity Bot - Server monitoriong page</title>
+ <title>Apache Ignite Teamcity Bot - Server monitoring page</title>
<link rel="icon" href="img/leaf-icon-png-7066.png">
<link rel="stylesheet"
href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css";>
@@ -54,6 +54,12 @@
error: showErrInLoadStatus
});
+ $.ajax({
+ url: "rest/monitoring/appLogSummaryLink",
+ success: showAppLogSummaryLink,
+ error: showErrInLoadStatus
+ });
+
loadPofilingData();
$.ajax({
@@ -78,21 +84,35 @@
res += "<th>Count</th>";
res += "<th>End</th>";
res += "<th>Result</th>";
+ res += "<th>Actions</th>";
res += "</tr>";
for (var i = 0; i < result.length; i++) {
var task = result[i];
+ var logUrl = "monitoring-log.html?startTs=" +
encodeURIComponent(task.startTs)
+ + "&endTs=" + encodeURIComponent(task.endTs || 0)
+ + "&name=" + encodeURIComponent(task.name);
+
res += "<tr>";
- res += "<td>" + task.name + "</td>";
- res += "<td>" + task.start + "</td>";
- res += "<td>" + task.count + "</td>";
- res += "<td>" + task.end + "</td>";
- res += "<td>" + task.result + "</td>";
+ res += "<td>" + escapeHtml(task.name) + "</td>";
+ res += "<td>" + escapeHtml(task.start) + "</td>";
+ res += "<td>" + escapeHtml(task.count) + "</td>";
+ res += "<td>" + escapeHtml(task.end) + "</td>";
+ res += "<td>" + escapeHtml(task.result) + "</td>";
+ res += "<td><a target='_blank' href='" + logUrl +
"'>Warnings/errors</a></td>";
res += "</tr>";
}
res += "</table>";
$("#tasks").html(res);
}
+ function showAppLogSummaryLink(result) {
+ var logUrl = "monitoring-log.html?startTs=" +
encodeURIComponent(result.startTs)
+ + "&endTs=0"
+ + "&name=" + encodeURIComponent(result.name);
+
+ $("#appLogSummary").html("<a target='_blank' href='" + logUrl +
"'>Application warnings/errors since start</a>");
+ }
+
/**
* @param result org.apache.ignite.ci.web.rest.monitoring.HotSpot
*/
@@ -107,10 +127,10 @@
for (var i = 0; i < result.length; i++) {
var inv = result[i];
res += "<tr>";
- res += "<td>" + inv.method + "</td>";
- res += "<td>" + inv.duration + "</td>";
- res += "<td>" + inv.count + "</td>";
- res += "<td>" + inv.avgDuration + "</td>";
+ res += "<td>" + escapeHtml(inv.method) + "</td>";
+ res += "<td>" + escapeHtml(inv.duration) + "</td>";
+ res += "<td>" + escapeHtml(inv.count) + "</td>";
+ res += "<td>" + escapeHtml(inv.avgDuration) + "</td>";
res += "</tr>";
}
res += "</table>";
@@ -127,9 +147,9 @@
for (var i = 0; i < result.length; i++) {
var inv = result[i];
res += "<tr>";
- res += "<td>" + inv.name + "</td>";
- res += "<td>" + inv.size + "</td>";
- res += "<td>" + inv.parts + "</td>";
+ res += "<td>" + escapeHtml(inv.name) + "</td>";
+ res += "<td>" + escapeHtml(inv.size) + "</td>";
+ res += "<td>" + escapeHtml(inv.parts) + "</td>";
res += "</tr>";
}
$("#caches").html(res);
@@ -177,9 +197,19 @@
return false;
}
+ function escapeHtml(str) {
+ return String(str || "")
+ .replace(/&/g, "&")
+ .replace(/</g, "<")
+ .replace(/>/g, ">")
+ .replace(/"/g, """)
+ .replace(/'/g, "'");
+ }
+
</script>
Tasks Monitoring Data:
+<div id="appLogSummary" style="font-family: monospace"></div>
<div id="tasks" style="font-family: monospace"></div>
<br>
@@ -210,4 +240,4 @@ Tasks Monitoring Data:
<div id="version"></div>
</body>
-</html>
\ No newline at end of file
+</html>
diff --git
a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/exeption/ServiceUnavailableException.java
b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/exeption/ServiceUnavailableException.java
new file mode 100644
index 00000000..95327c09
--- /dev/null
+++
b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/exeption/ServiceUnavailableException.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tcbot.common.exeption;
+
+import javax.annotation.Nullable;
+
+/**
+ * The service is temporarily unable to handle the request.
+ *
+ * This exception is thrown in case HTTP 503-Service Unavailable is returned.
+ */
+public class ServiceUnavailableException extends RuntimeException {
+ /** Service URL. */
+ private final String url;
+
+ /** HTTP status code. */
+ private final int responseCode;
+
+ /** Retry-After delay in milliseconds, or {@code -1} if it is absent or
cannot be parsed. */
+ private final long retryAfterMs;
+
+ /**
+ * @param msg Message.
+ * @param url Service URL.
+ * @param responseCode HTTP status code.
+ * @param retryAfterMs Retry-After delay in milliseconds, or {@code -1}.
+ */
+ public ServiceUnavailableException(String msg, String url, int
responseCode, long retryAfterMs) {
+ this(msg, url, responseCode, retryAfterMs, null);
+ }
+
+ /**
+ * @param msg Message.
+ * @param url Service URL.
+ * @param responseCode HTTP status code.
+ * @param retryAfterMs Retry-After delay in milliseconds, or {@code -1}.
+ * @param cause Cause.
+ */
+ public ServiceUnavailableException(String msg, String url, int
responseCode, long retryAfterMs,
+ @Nullable Throwable cause) {
+ super(msg, cause);
+
+ this.url = url;
+ this.responseCode = responseCode;
+ this.retryAfterMs = retryAfterMs;
+ }
+
+ /**
+ * @return Service URL.
+ */
+ public String url() {
+ return url;
+ }
+
+ /**
+ * @return HTTP status code.
+ */
+ public int responseCode() {
+ return responseCode;
+ }
+
+ /**
+ * @return Retry-After delay in milliseconds, or {@code -1}.
+ */
+ public long retryAfterMs() {
+ return retryAfterMs;
+ }
+}
diff --git
a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/interceptor/MonitoredTaskInterceptor.java
b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/interceptor/MonitoredTaskInterceptor.java
index 20a5a858..bf8b00ea 100644
---
a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/interceptor/MonitoredTaskInterceptor.java
+++
b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/interceptor/MonitoredTaskInterceptor.java
@@ -42,6 +42,8 @@ import static
org.apache.ignite.tcbot.common.util.TimeUtil.timestampForLogsSimpl
public class MonitoredTaskInterceptor implements MethodInterceptor,
AutoCloseable {
private final ConcurrentMap<String, Invocation> totalTime = new
ConcurrentSkipListMap<>();
+ private final long startedTs = System.currentTimeMillis();
+
private FileWriter fileWriter;
private final AtomicBoolean init = new AtomicBoolean();
@@ -77,7 +79,7 @@ public class MonitoredTaskInterceptor implements
MethodInterceptor, AutoCloseabl
private final AtomicInteger callsCnt = new AtomicInteger();
/** Name and full key for monitored task. */
- private String name;
+ private final String name;
Invocation(String name) {
this.name = name;
@@ -104,6 +106,20 @@ public class MonitoredTaskInterceptor implements
MethodInterceptor, AutoCloseabl
return callsCnt.get();
}
+ /**
+ * @return Last observed start timestamp.
+ */
+ public long startTs() {
+ return lastStartTs.get();
+ }
+
+ /**
+ * @return Last observed end timestamp.
+ */
+ public long endTs() {
+ return lastEndTs.get();
+ }
+
/**
* @return time printable of last observed start time of the task.
*/
@@ -250,10 +266,7 @@ public class MonitoredTaskInterceptor implements
MethodInterceptor, AutoCloseabl
}
}
- if (annotation.log())
- log = true;
- else
- log = false;
+ log = annotation.log();
}
else {
@@ -268,4 +281,11 @@ public class MonitoredTaskInterceptor implements
MethodInterceptor, AutoCloseabl
public Collection<Invocation> getList() {
return Collections.unmodifiableCollection(totalTime.values());
}
+
+ /**
+ * @return Interceptor creation timestamp, close enough to app startup for
monitoring purposes.
+ */
+ public long startedTs() {
+ return startedTs;
+ }
}
diff --git
a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/util/HttpUtil.java
b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/util/HttpUtil.java
index 057981e4..837b1ba2 100644
---
a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/util/HttpUtil.java
+++
b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/util/HttpUtil.java
@@ -31,12 +31,17 @@ import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
+import java.time.Duration;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.ignite.tcbot.common.exeption.ServiceUnauthorizedException;
import org.apache.ignite.tcbot.common.exeption.ServiceBadRequestException;
import org.apache.ignite.tcbot.common.exeption.ServiceConflictException;
+import org.apache.ignite.tcbot.common.exeption.ServiceUnavailableException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,7 +96,14 @@ public class HttpUtil {
con.setRequestProperty("Keep-Alive", "header");
con.setRequestProperty("accept-charset",
StandardCharsets.UTF_8.toString());
- int resCode = con.getResponseCode();
+ int resCode;
+
+ try {
+ resCode = con.getResponseCode();
+ }
+ catch (IOException e) {
+ throw new IOException("Failed GET request [host=" + obj.getHost()
+ ", url=" + url + ']', e);
+ }
logger.info(Thread.currentThread().getName() + ": Required: " +
started.elapsed(TimeUnit.MILLISECONDS)
+ "ms : Sending 'GET' request to : " + url + " Response: " +
resCode);
@@ -230,6 +242,11 @@ public class HttpUtil {
throw new ServiceConflictException("Service " + con.getURL() + "
returned Conflict Response Code:\n"
+ diagnostic);
+ if (resCode == 503)
+ throw new ServiceUnavailableException("Service " + con.getURL() +
" (host=" + con.getURL().getHost()
+ + ") returned Service Unavailable Response Code : " + resCode
+ ":\n" + diagnostic,
+ con.getURL().toString(), resCode,
retryAfterMs(con.getHeaderField("Retry-After")));
+
throw new IllegalStateException("Service " + con.getURL() + " returned
Invalid Response Code : " + resCode + ":\n"
+ diagnostic);
}
@@ -253,6 +270,9 @@ public class HttpUtil {
if (authDiagnostic != null)
res.append(authDiagnostic).append('\n');
+ res.append("Response URL: ").append(con.getURL()).append('\n');
+ res.append("Response host:
").append(con.getURL().getHost()).append('\n');
+
appendHeaderIfPresent(res, con, "WWW-Authenticate");
appendHeaderIfPresent(res, con, "X-GitHub-Request-Id");
appendHeaderIfPresent(res, con, "X-RateLimit-Limit");
@@ -282,6 +302,33 @@ public class HttpUtil {
res.append(name).append(": ").append(val).append('\n');
}
+ /**
+ * @param retryAfter Retry-After header value.
+ */
+ private static long retryAfterMs(@Nullable String retryAfter) {
+ if (retryAfter == null || retryAfter.trim().isEmpty())
+ return -1;
+
+ String trimmed = retryAfter.trim();
+
+ try {
+ return TimeUnit.SECONDS.toMillis(Long.parseLong(trimmed));
+ }
+ catch (NumberFormatException ignored) {
+ // Retry-After also allows HTTP-date.
+ }
+
+ try {
+ Duration delay = Duration.between(ZonedDateTime.now(),
+ ZonedDateTime.parse(trimmed,
DateTimeFormatter.RFC_1123_DATE_TIME));
+
+ return Math.max(0, delay.toMillis());
+ }
+ catch (DateTimeParseException ignored) {
+ return -1;
+ }
+ }
+
/**
* Send POST request to the GitHub url.
*
diff --git
a/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java
b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java
index 21214ef2..14b2b3e3 100644
---
a/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java
+++
b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java
@@ -16,6 +16,8 @@
*/
package org.apache.ignite.githubignited;
+import java.io.FileNotFoundException;
+import java.io.UncheckedIOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -185,6 +187,9 @@ class GitHubConnIgnitedImpl implements IGitHubConnIgnited {
int cntSaved = savePrsChunk(ghData);
int totalChecked = ghData.size();
+ if (fullReindex)
+ collectPullRequestIds(ghData, actualPrs);
+
while (outLinkNext.get() != null) {
String nextPageUrl = outLinkNext.get();
ghData = conn.getPullRequestsPage(nextPageUrl, outLinkNext);
@@ -192,11 +197,8 @@ class GitHubConnIgnitedImpl implements IGitHubConnIgnited {
cntSaved += savedThisChunk;
totalChecked += ghData.size();
- if (fullReindex) {
- actualPrs.addAll(ghData.stream()
- .map(PullRequest::getNumber)
- .collect(Collectors.toSet()));
- }
+ if (fullReindex)
+ collectPullRequestIds(ghData, actualPrs);
if (!fullReindex && savedThisChunk == 0)
break;
@@ -213,14 +215,85 @@ class GitHubConnIgnitedImpl implements IGitHubConnIgnited
{
@SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
@MonitoredTask(name = "Check Outdated PRs(srv)", nameExtArgsIndexes = {0})
protected String refreshOutdatedPrs(String srvId, Set<Integer> actualPrs) {
- final long cnt = StreamSupport.stream(prCache.spliterator(), false)
+ List<Integer> outdatedPrs =
StreamSupport.stream(prCache.spliterator(), false)
.filter(entry -> entry.getKey() >> 32 == srvIdMaskHigh)
.filter(entry ->
PullRequest.OPEN.equals(entry.getValue().getState()))
.filter(entry -> !actualPrs.contains(entry.getValue().getNumber()))
- .peek(entry -> prCache.put(entry.getKey(),
conn.getPullRequest(entry.getValue().getNumber())))
- .count();
+ .map(entry -> entry.getValue().getNumber())
+ .collect(Collectors.toList());
+
+ int cntUpdated = 0;
+ int cntRemoved = 0;
+
+ for (Integer prNum : outdatedPrs) {
+ PullRequest pr = getPullRequestOrRemoveIfNotFound(prNum);
+
+ if (pr == null)
+ cntRemoved++;
+ else {
+ prCache.put(prNumberToCacheKey(prNum), pr);
+ cntUpdated++;
+ }
+ }
+
+ return "PRs updated for " + srvId + ": " + cntUpdated + ", removed
stale: " + cntRemoved +
+ " from " + prCache.size();
+ }
+
+ /**
+ * @param prs Pull requests.
+ * @param actualPrs Actual pull request ids.
+ */
+ private void collectPullRequestIds(List<PullRequest> prs, Set<Integer>
actualPrs) {
+ actualPrs.addAll(prs.stream()
+ .map(PullRequest::getNumber)
+ .collect(Collectors.toSet()));
+ }
+
+ /**
+ * @param branches GitHub branches.
+ * @param actualPrs Actual pull request ids.
+ */
+ private void collectPullRequestIdsFromBranches(List<GitHubBranchShort>
branches, Set<Integer> actualPrs) {
+ actualPrs.addAll(branches.stream()
+ .map(GitHubBranchShort::name)
+ .map(IGitHubConnection::convertBranchToPrId)
+ .filter(prId -> prId != null)
+ .collect(Collectors.toSet()));
+ }
+
+ /**
+ * @param prNum PR number.
+ */
+ @Nullable
+ private PullRequest getPullRequestOrRemoveIfNotFound(Integer prNum) {
+ try {
+ return conn.getPullRequest(prNum);
+ }
+ catch (UncheckedIOException e) {
+ if (!isNotFound(e))
+ throw e;
+
+ long cacheKey = prNumberToCacheKey(prNum);
+
+ logger.info("Removing stale GitHub PR from cache [srv={}, pr={}]",
srvCode, prNum);
- return "PRs updated for " + srvId + ": " + cnt + " from " +
prCache.size();
+ prCache.remove(cacheKey);
+
+ return null;
+ }
+ }
+
+ /**
+ * @param e Exception.
+ */
+ private boolean isNotFound(UncheckedIOException e) {
+ for (Throwable th = e; th != null; th = th.getCause()) {
+ if (th instanceof FileNotFoundException)
+ return true;
+ }
+
+ return false;
}
/**
@@ -264,25 +337,45 @@ class GitHubConnIgnitedImpl implements IGitHubConnIgnited
{
protected String runActualizeBranches(String srvId, boolean fullReindex) {
AtomicReference<String> outLinkNext = new AtomicReference<>();
- List<GitHubBranchShort> ghData = conn.getBranchesPage(null,
outLinkNext);
-
Set<Integer> actualPrs = new HashSet<>();
- int cntSaved = saveBranchesChunk(ghData);
- int totalChecked = ghData.size();
- while (outLinkNext.get() != null) {
- String nextPageUrl = outLinkNext.get();
- ghData = conn.getBranchesPage(nextPageUrl, outLinkNext);
- int savedThisChunk = saveBranchesChunk(ghData);
- cntSaved += savedThisChunk;
- totalChecked += ghData.size();
+ int cntSaved = 0;
+ int totalChecked = 0;
+ int page = 1;
+ String nextPageUrl = null;
- if (!fullReindex && savedThisChunk == 0)
- break;
- }
+ try {
+ List<GitHubBranchShort> ghData = conn.getBranchesPage(null,
outLinkNext);
- if (fullReindex)
- refreshOutdatedPrs(srvId, actualPrs);
+ cntSaved = saveBranchesChunk(ghData);
+ totalChecked = ghData.size();
+ if (fullReindex)
+ collectPullRequestIdsFromBranches(ghData, actualPrs);
+
+ while (outLinkNext.get() != null) {
+ nextPageUrl = outLinkNext.get();
+ page++;
+ ghData = conn.getBranchesPage(nextPageUrl, outLinkNext);
+ int savedThisChunk = saveBranchesChunk(ghData);
+ cntSaved += savedThisChunk;
+ totalChecked += ghData.size();
+
+ if (fullReindex)
+ collectPullRequestIdsFromBranches(ghData, actualPrs);
+
+ if (!fullReindex && savedThisChunk == 0)
+ break;
+ }
+
+ if (fullReindex)
+ refreshOutdatedPrs(srvId, actualPrs);
+ }
+ catch (UncheckedIOException e) {
+ throw new UncheckedIOException("Failed to actualize GitHub
branches [srv=" + srvId +
+ ", fullReindex=" + fullReindex + ", page=" + page + ",
totalChecked=" + totalChecked +
+ ", cntSaved=" + cntSaved + ", actualPrs=" + actualPrs.size() +
+ ", nextPageUrl=" + nextPageUrl + ", cause=" + e.getMessage() +
']', e.getCause());
+ }
return "Entries saved " + cntSaved + " Branches checked " +
totalChecked;
}
diff --git
a/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java
b/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java
index 832cf2f8..21fb7754 100644
---
a/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java
+++
b/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java
@@ -36,11 +36,16 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UncheckedIOException;
+import java.net.ConnectException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.StringTokenizer;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.LockSupport;
@@ -58,6 +63,18 @@ class GitHubConnectionImpl implements IGitHubConnection {
/** Service (server) code. */
private String srvCode;
+ /** GitHub read attempts. */
+ private static final int READ_ATTEMPTS = 3;
+
+ /** Initial retry backoff. */
+ private static final long INITIAL_RETRY_BACKOFF_MS = 500;
+
+ /** Retry jitter. */
+ private static final long RETRY_JITTER_MS = 250;
+
+ /** Max retry backoff. */
+ private static final long MAX_RETRY_BACKOFF_MS =
TimeUnit.SECONDS.toMillis(30);
+
private static AtomicLong lastRq = new AtomicLong();
/**
@@ -106,14 +123,31 @@ class GitHubConnectionImpl implements IGitHubConnection {
String pr = gitApiUrl + "pulls/" + id;
- try (InputStream is = sendGetToGit(pr, null)) {
- InputStreamReader reader = new InputStreamReader(is);
+ for (int attempt = 1; attempt <= READ_ATTEMPTS; attempt++) {
+ try (InputStream is = sendGetToGit(pr, null)) {
+ InputStreamReader reader = new InputStreamReader(is);
- return new Gson().fromJson(reader, PullRequest.class);
- }
- catch (IOException e) {
- throw new UncheckedIOException(e);
+ return new Gson().fromJson(reader, PullRequest.class);
+ }
+ catch (IOException e) {
+ if (shouldRetry(e, attempt)) {
+ long backoffMs = retryBackoffMs(attempt);
+
+ logger.warn("Failed to read GitHub pull request, will
retry " +
+ "[srv={}, pr={}, url={}, attempt={}/{}, backoffMs={}]",
+ srvCode, id, pr, attempt, READ_ATTEMPTS, backoffMs, e);
+
+
LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(backoffMs));
+
+ continue;
+ }
+
+ throw new UncheckedIOException("Failed to read GitHub pull
request [srv=" + srvCode +
+ ", pr=" + id + ", url=" + pr + ", attempt=" + attempt +
'/' + READ_ATTEMPTS + ']', e);
+ }
}
+
+ throw new IllegalStateException("Unreachable");
}
/** {@inheritDoc} */
@@ -176,25 +210,75 @@ class GitHubConnectionImpl implements IGitHubConnection {
public <T> List<T> readOnePage(@Nullable AtomicReference<String>
outLinkNext,
String url, HashMap<String, String> rspHeaders,
TypeToken<ArrayList<T>> typeTok) {
- try (InputStream stream = sendGetToGit(url, rspHeaders)) {
- InputStreamReader reader = new InputStreamReader(stream);
- List<T> list = new Gson().fromJson(reader, typeTok.getType());
- String link = rspHeaders.get("Link");
+ for (int attempt = 1; attempt <= READ_ATTEMPTS; attempt++) {
+ if (rspHeaders.containsKey("Link"))
+ rspHeaders.put("Link", null);
+
+ try (InputStream stream = sendGetToGit(url, rspHeaders)) {
+ InputStreamReader reader = new InputStreamReader(stream);
+ List<T> list = new Gson().fromJson(reader, typeTok.getType());
+ String link = rspHeaders.get("Link");
- if (link != null) {
- String nextLink = parseNextLinkFromLinkRspHeader(link);
+ if (link != null) {
+ String nextLink = parseNextLinkFromLinkRspHeader(link);
- if (nextLink != null && outLinkNext != null)
- outLinkNext.set(nextLink);
+ if (nextLink != null && outLinkNext != null)
+ outLinkNext.set(nextLink);
+ }
+
+ logger.info("Processing Github link: " + link);
+
+ return list;
}
+ catch (IOException e) {
+ if (shouldRetry(e, attempt)) {
+ long backoffMs = retryBackoffMs(attempt);
+
+ logger.warn("Failed to read GitHub page, will retry
[srv={}, url={}, attempt={}/{}, backoffMs={}]",
+ srvCode, url, attempt, READ_ATTEMPTS, backoffMs, e);
+
+
LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(backoffMs));
- logger.info("Processing Github link: " + link);
+ continue;
+ }
- return list;
+ throw new UncheckedIOException("Failed to read GitHub page
[srv=" + srvCode +
+ ", url=" + url + ", link=" + rspHeaders.get("Link") +
+ ", attempt=" + attempt + '/' + READ_ATTEMPTS + ']', e);
+ }
}
- catch (IOException e) {
- throw new UncheckedIOException(e);
+
+ throw new IllegalStateException("Unreachable");
+ }
+
+ /**
+ * @param e Exception.
+ * @param attempt Attempt.
+ */
+ private boolean shouldRetry(IOException e, int attempt) {
+ return attempt < READ_ATTEMPTS && isTemporaryTransportFailure(e);
+ }
+
+ /**
+ * @param e Exception.
+ */
+ private boolean isTemporaryTransportFailure(Throwable e) {
+ for (Throwable th = e; th != null; th = th.getCause()) {
+ if (th instanceof ConnectException || th instanceof
SocketException || th instanceof SocketTimeoutException)
+ return true;
}
+
+ return false;
+ }
+
+ /**
+ * @param attempt Attempt.
+ */
+ private long retryBackoffMs(int attempt) {
+ long base = INITIAL_RETRY_BACKOFF_MS << (attempt - 1);
+ long backoff = base +
ThreadLocalRandom.current().nextLong(RETRY_JITTER_MS + 1);
+
+ return Math.min(backoff, MAX_RETRY_BACKOFF_MS);
}
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefSync.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefSync.java
index 05cac6fa..dd253dbe 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefSync.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefSync.java
@@ -16,6 +16,7 @@
*/
package org.apache.ignite.tcignited.buildref;
+import java.io.UncheckedIOException;
import java.time.Duration;
import java.util.Collection;
import java.util.List;
@@ -77,79 +78,93 @@ public class BuildRefSync {
ITeamcityConn conn) {
AtomicReference<String> outLinkNext = new AtomicReference<>();
- List<BuildRef> tcDataFirstPage = conn.getBuildRefsPage(null,
outLinkNext);
long start = System.currentTimeMillis();
int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId);
- Set<Long> buildsUpdated = buildRefDao.saveChunk(srvIdMaskHigh,
tcDataFirstPage);
- int totalUpdated = buildsUpdated.size();
- fatBuildSync.scheduleBuildsLoad(conn,
cacheKeysToBuildIds(buildsUpdated));
-
- int totalChecked = tcDataFirstPage.size();
+ int page = 1;
+ String nextPageUrl = null;
+ int totalUpdated = 0;
+ int totalChecked = 0;
int neededToFind = 0;
- if (mandatoryToReload != null) {
- neededToFind = mandatoryToReload.size();
+ long lastTimeUpdateFound = System.currentTimeMillis();
+ boolean timeoutForNewBuild = false;
-
tcDataFirstPage.stream().map(BuildRef::getId).forEach(mandatoryToReload::remove);
- }
+ try {
+ List<BuildRef> tcDataFirstPage = conn.getBuildRefsPage(null,
outLinkNext);
+ Set<Long> buildsUpdated = buildRefDao.saveChunk(srvIdMaskHigh,
tcDataFirstPage);
+ totalUpdated = buildsUpdated.size();
+ fatBuildSync.scheduleBuildsLoad(conn,
cacheKeysToBuildIds(buildsUpdated));
- if (syncMode == SyncMode.ULTRAFAST && isEmpty(mandatoryToReload)) {
- return "Entries saved " +
- totalUpdated +
- " Builds checked " +
- totalChecked +
- " Needed to find " +
- neededToFind +
- " remained to find " +
- mandatoryToReload.size();
- }
+ totalChecked = tcDataFirstPage.size();
+ if (mandatoryToReload != null) {
+ neededToFind = mandatoryToReload.size();
- long lastTimeUpdateFound = System.currentTimeMillis();
- long maxMsWithoutChanges = Duration.ofHours(1).toMillis();
+
tcDataFirstPage.stream().map(BuildRef::getId).forEach(mandatoryToReload::remove);
+ }
- //reason for end for full sync
- boolean timeoutForNewBuild = false;
- //reason for end for incremental sync: decrementing counter of builds
to find without modification to stop search.
- int buildsCntrToStop = INCREMENTAL_BUILDS_WO_MODIFICATION_TO_STOP;
-
- while (outLinkNext.get() != null) {
- String nextPageUrl = outLinkNext.get();
- outLinkNext.set(null);
- List<BuildRef> tcDataNextPage = conn.getBuildRefsPage(nextPageUrl,
outLinkNext);
- Set<Long> curChunkBuildsSaved =
buildRefDao.saveChunk(srvIdMaskHigh, tcDataNextPage);
- totalUpdated += curChunkBuildsSaved.size();
- fatBuildSync.scheduleBuildsLoad(conn,
cacheKeysToBuildIds(curChunkBuildsSaved));
-
- int savedCurChunk = curChunkBuildsSaved.size();
-
- totalChecked += tcDataNextPage.size();
- if (savedCurChunk != 0) {
- lastTimeUpdateFound = System.currentTimeMillis();
-
- buildsCntrToStop = INCREMENTAL_BUILDS_WO_MODIFICATION_TO_STOP;
- } else
- buildsCntrToStop -= tcDataNextPage.size();
-
- if (syncMode == SyncMode.ULTRAFAST && isEmpty(mandatoryToReload))
- break;
- else if (syncMode==SyncMode.FULL_REINDEX) {
- timeoutForNewBuild = System.currentTimeMillis() >
lastTimeUpdateFound + maxMsWithoutChanges;
- if (timeoutForNewBuild
- && totalChecked > MAX_INCREMENTAL_BUILDS_TO_CHECK)
- break;
+ if (syncMode == SyncMode.ULTRAFAST && isEmpty(mandatoryToReload)) {
+ return "Entries saved " +
+ totalUpdated +
+ " Builds checked " +
+ totalChecked +
+ " Needed to find " +
+ neededToFind +
+ " remained to find " +
+ mandatoryToReload.size();
}
- else {
- boolean noMandatoryBuildsLeft = isEmpty(mandatoryToReload);
- if (!noMandatoryBuildsLeft)
-
tcDataNextPage.stream().map(BuildRef::getId).forEach(mandatoryToReload::remove);
-
- if (buildsCntrToStop <= 0
- && (noMandatoryBuildsLeft || totalChecked >
MAX_INCREMENTAL_BUILDS_TO_CHECK)) {
- // There are no modification at current page, hopefully no
modifications at all
+
+ long maxMsWithoutChanges = Duration.ofHours(1).toMillis();
+
+ //reason for end for incremental sync: decrementing counter of
builds to find without modification to stop search.
+ int buildsCntrToStop = INCREMENTAL_BUILDS_WO_MODIFICATION_TO_STOP;
+
+ while (outLinkNext.get() != null) {
+ nextPageUrl = outLinkNext.get();
+ page++;
+ outLinkNext.set(null);
+ List<BuildRef> tcDataNextPage =
conn.getBuildRefsPage(nextPageUrl, outLinkNext);
+ Set<Long> curChunkBuildsSaved =
buildRefDao.saveChunk(srvIdMaskHigh, tcDataNextPage);
+ totalUpdated += curChunkBuildsSaved.size();
+ fatBuildSync.scheduleBuildsLoad(conn,
cacheKeysToBuildIds(curChunkBuildsSaved));
+
+ int savedCurChunk = curChunkBuildsSaved.size();
+
+ totalChecked += tcDataNextPage.size();
+ if (savedCurChunk != 0) {
+ lastTimeUpdateFound = System.currentTimeMillis();
+
+ buildsCntrToStop =
INCREMENTAL_BUILDS_WO_MODIFICATION_TO_STOP;
+ } else
+ buildsCntrToStop -= tcDataNextPage.size();
+
+ if (syncMode == SyncMode.ULTRAFAST &&
isEmpty(mandatoryToReload))
break;
+ else if (syncMode==SyncMode.FULL_REINDEX) {
+ timeoutForNewBuild = System.currentTimeMillis() >
lastTimeUpdateFound + maxMsWithoutChanges;
+ if (timeoutForNewBuild
+ && totalChecked > MAX_INCREMENTAL_BUILDS_TO_CHECK)
+ break;
+ }
+ else {
+ boolean noMandatoryBuildsLeft = isEmpty(mandatoryToReload);
+ if (!noMandatoryBuildsLeft)
+
tcDataNextPage.stream().map(BuildRef::getId).forEach(mandatoryToReload::remove);
+
+ if (buildsCntrToStop <= 0
+ && (noMandatoryBuildsLeft || totalChecked >
MAX_INCREMENTAL_BUILDS_TO_CHECK)) {
+ // There are no modification at current page,
hopefully no modifications at all
+ break;
+ }
}
}
}
+ catch (UncheckedIOException e) {
+ throw new UncheckedIOException("Failed to actualize TeamCity build
refs [srv=" + srvId +
+ ", syncMode=" + syncMode + ", page=" + page + ",
totalChecked=" + totalChecked +
+ ", totalUpdated=" + totalUpdated + ", neededToFind=" +
neededToFind +
+ ", remainedToFind=" + (mandatoryToReload == null ? 0 :
mandatoryToReload.size()) +
+ ", nextPageUrl=" + nextPageUrl + ", cause=" + e.getMessage() +
']', e.getCause());
+ }
StringBuilder sb = new StringBuilder();
sb.append("Entries saved ");
diff --git
a/tcbot-teamcity/src/main/java/org/apache/ignite/tcservice/TeamcityServiceConnection.java
b/tcbot-teamcity/src/main/java/org/apache/ignite/tcservice/TeamcityServiceConnection.java
index 23229658..84339e3a 100644
---
a/tcbot-teamcity/src/main/java/org/apache/ignite/tcservice/TeamcityServiceConnection.java
+++
b/tcbot-teamcity/src/main/java/org/apache/ignite/tcservice/TeamcityServiceConnection.java
@@ -24,12 +24,20 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.io.UncheckedIOException;
+import java.net.ConnectException;
+import java.net.MalformedURLException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.URL;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.SortedSet;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.LockSupport;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -40,6 +48,7 @@ import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.common.conf.TcBotWorkDir;
import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.tcbot.common.exeption.ServiceConflictException;
+import org.apache.ignite.tcbot.common.exeption.ServiceUnavailableException;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.util.HttpUtil;
import org.apache.ignite.tcservice.http.ITeamcityHttpConnection;
@@ -83,6 +92,18 @@ public class TeamcityServiceConnection implements ITeamcity {
/** Teamcity http connection. */
@Inject private ITeamcityHttpConnection teamcityHttpConn;
+ /** TeamCity GET attempts for temporary transport failures. */
+ private static final int GET_ATTEMPTS = 3;
+
+ /** Initial retry backoff. */
+ private static final long INITIAL_RETRY_BACKOFF_MS = 500;
+
+ /** Retry jitter. */
+ private static final long RETRY_JITTER_MS = 250;
+
+ /** Max retry backoff. */
+ private static final long MAX_RETRY_BACKOFF_MS =
TimeUnit.SECONDS.toMillis(30);
+
@Inject private IDataSourcesConfigSupplier cfg;
private String srvCode;
@@ -283,18 +304,106 @@ public class TeamcityServiceConnection implements
ITeamcity {
* @throws UncheckedIOException in case communication failed.
*/
private <T> T sendGetXmlParseJaxb(String url, Class<T> rootElem) {
- try {
- try (InputStream inputStream =
teamcityHttpConn.sendGet(basicAuthTok, url)) {
- final InputStreamReader reader = new
InputStreamReader(inputStream);
+ for (int attempt = 1; attempt <= GET_ATTEMPTS; attempt++) {
+ try {
+ try (InputStream inputStream =
teamcityHttpConn.sendGet(basicAuthTok, url)) {
+ final InputStreamReader reader = new
InputStreamReader(inputStream);
+
+ return loadXml(rootElem, reader);
+ }
+ }
+ catch (IOException e) {
+ if (shouldRetry(e, attempt)) {
+ long backoffMs = retryBackoffMs(attempt, -1);
+
+ logger.warn("Failed to read TeamCity XML, will retry " +
+ "[srv={}, host={}, url={}, root={}, attempt={}/{},
backoffMs={}]",
+ srvCode, host(url), url, rootElem.getSimpleName(),
attempt, GET_ATTEMPTS, backoffMs, e);
+
+
LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(backoffMs));
- return loadXml(rootElem, reader);
+ continue;
+ }
+
+ throw new UncheckedIOException("Failed to read TeamCity XML
[srv=" + srvCode +
+ ", host=" + host(url) + ", url=" + url + ", root=" +
rootElem.getName() +
+ ", attempt=" + attempt + '/' + GET_ATTEMPTS + ']', e);
+ }
+ catch (ServiceUnavailableException e) {
+ if (shouldRetryServiceUnavailable(attempt)) {
+ long backoffMs = retryBackoffMs(attempt, e.retryAfterMs());
+
+ logger.warn("TeamCity service unavailable, will retry " +
+ "[srv={}, host={}, url={}, root={}, attempt={}/{},
status={}, retryAfterMs={}, backoffMs={}]",
+ srvCode, host(url), url, rootElem.getSimpleName(),
attempt, GET_ATTEMPTS,
+ e.responseCode(), e.retryAfterMs(), backoffMs, e);
+
+
LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(backoffMs));
+
+ continue;
+ }
+
+ throw new ServiceUnavailableException("Failed to read TeamCity
XML after HTTP "
+ + e.responseCode() + " [srv=" + srvCode + ", host=" +
host(url) + ", url=" + url
+ + ", root=" + rootElem.getName() + ", attempt=" + attempt
+ '/' + GET_ATTEMPTS + "]\n"
+ + e.getMessage(), e.url(), e.responseCode(),
e.retryAfterMs(), e);
+ }
+ catch (JAXBException e) {
+ throw ExceptionUtil.propagateException(e);
}
}
- catch (IOException e) {
- throw new UncheckedIOException(e);
+
+ throw new IllegalStateException("Unreachable");
+ }
+
+ /**
+ * @param e Exception.
+ * @param attempt Attempt.
+ */
+ private boolean shouldRetry(IOException e, int attempt) {
+ return attempt < GET_ATTEMPTS && isTemporaryTransportFailure(e);
+ }
+
+ /**
+ * @param attempt Attempt.
+ */
+ private boolean shouldRetryServiceUnavailable(int attempt) {
+ return attempt < GET_ATTEMPTS;
+ }
+
+ /**
+ * @param attempt Attempt.
+ * @param retryAfterMs Retry-After delay, or {@code -1}.
+ */
+ private long retryBackoffMs(int attempt, long retryAfterMs) {
+ long base = INITIAL_RETRY_BACKOFF_MS << (attempt - 1);
+ long backoff = base +
ThreadLocalRandom.current().nextLong(RETRY_JITTER_MS + 1);
+ long retryDelay = retryAfterMs >= 0 ? Math.max(backoff, retryAfterMs)
: backoff;
+
+ return Math.min(retryDelay, MAX_RETRY_BACKOFF_MS);
+ }
+
+ /**
+ * @param e Exception.
+ */
+ private boolean isTemporaryTransportFailure(Throwable e) {
+ for (Throwable th = e; th != null; th = th.getCause()) {
+ if (th instanceof ConnectException || th instanceof
SocketException || th instanceof SocketTimeoutException)
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @param url URL.
+ */
+ private String host(String url) {
+ try {
+ return new URL(url).getHost();
}
- catch (JAXBException e) {
- throw ExceptionUtil.propagateException(e);
+ catch (MalformedURLException ignored) {
+ return "<unknown>";
}
}
diff --git
a/tcbot-teamcity/src/main/java/org/apache/ignite/tcservice/http/ITeamcityHttpConnection.java
b/tcbot-teamcity/src/main/java/org/apache/ignite/tcservice/http/ITeamcityHttpConnection.java
index 9b222b58..cb924706 100644
---
a/tcbot-teamcity/src/main/java/org/apache/ignite/tcservice/http/ITeamcityHttpConnection.java
+++
b/tcbot-teamcity/src/main/java/org/apache/ignite/tcservice/http/ITeamcityHttpConnection.java
@@ -18,6 +18,7 @@
package org.apache.ignite.tcservice.http;
import org.apache.ignite.tcbot.common.exeption.ServiceConflictException;
+import org.apache.ignite.tcbot.common.exeption.ServiceUnavailableException;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -29,6 +30,7 @@ public interface ITeamcityHttpConnection {
* @param url Url.
* @throws FileNotFoundException If not found (404) was returned from
service.
* @throws ServiceConflictException If conflict (409) was returned from
service.
+ * @throws ServiceUnavailableException If service unavailable (503) was
returned from service.
* @throws IllegalStateException if some unexpected HTTP error returned.
*/
public InputStream sendGet(String basicAuthTok, String url) throws
IOException;
