This is an automated email from the ASF dual-hosted git repository.
dspavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 2f1442e1 IGNITE-28638 performance issues localization for the TCbot
(#210)
2f1442e1 is described below
commit 2f1442e1c3a610d2981e23c0fbba60491fd97a0c
Author: ignitetcbot <[email protected]>
AuthorDate: Fri May 8 18:53:21 2026 +0300
IGNITE-28638 performance issues localization for the TCbot (#210)
Added REST request timing monitoring and slow-operation budget logs for PR
visa and tracked branch flows. Optimized several expensive paths by reusing PR
branch info, caching ticket lookups by key, using branch sets for lookup, and
grouping queued/running build counts per branch instead of requesting them per
suite.
Also added local persistence compatibility test coverage and minor cleanup
around visibility, servlet mapping, log directory creation, and monitoring UI
rendering.
Co-authored-by: Dmitriy Pavlov <[email protected]>
---
.gitignore | 3 +-
.../org/apache/ignite/ci/db/Ignite2Configurer.java | 2 +-
.../tcbot/visa/TcBotTriggerAndSignOffService.java | 243 ++++-
.../apache/ignite/ci/web/TcApplicationResCfg.java | 2 +
.../ci/web/rest/monitoring/MonitoringService.java | 20 +
.../ignite/ci/web/rest/monitoring/RequestStat.java | 33 +
.../monitoring/RequestTiming.java} | 44 +-
.../rest/monitoring/RestRequestTimingFilter.java | 75 ++
.../rest/monitoring/RestRequestTimingStorage.java | 156 +++
.../rest/tracked/GetTrackedBranchTestResults.java | 39 +-
ignite-tc-helper-web/src/main/webapp/current.html | 25 +-
.../src/main/webapp/monitoring.html | 65 +-
.../monitoring/MonitoringServiceSecurityTest.java | 62 ++
migrator/build.gradle | 17 +
.../LegacyPersistentStorageCompatibilityTest.java | 1120 ++++++++++++++++++++
migrator/src/test/resources/logback-test.xml | 11 +
.../tcbot/engine/chain/BuildChainProcessor.java | 69 +-
.../tcbot/engine/pr/BranchTicketMatcher.java | 37 +-
.../tracked/TrackedBranchChainsProcessor.java | 153 ++-
19 files changed, 2070 insertions(+), 106 deletions(-)
diff --git a/.gitignore b/.gitignore
index 276b8f61..dc3fbd97 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,4 +34,5 @@
hs_err_pid*
/conf/branches-prom.json
/ignite-tc-helper-web/ignite/**
-/ignite-tc-helper-web/src/test/tmp/**
\ No newline at end of file
+/ignite-tc-helper-web/src/test/tmp/**
+/migrator/src/test/work/**
\ No newline at end of file
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/Ignite2Configurer.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/Ignite2Configurer.java
index 26522d27..abe609be 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/Ignite2Configurer.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/Ignite2Configurer.java
@@ -123,7 +123,7 @@ public class Ignite2Configurer {
return regConf;
}
- static DataStorageConfiguration
getDataStorageConfiguration(DataRegionConfiguration regConf) {
+ public static DataStorageConfiguration
getDataStorageConfiguration(DataRegionConfiguration regConf) {
return new DataStorageConfiguration()
// .setWalCompactionEnabled(true)
.setWalMode(WALMode.LOG_ONLY)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java
index b5133514..021069fa 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/visa/TcBotTriggerAndSignOffService.java
@@ -31,8 +31,12 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
@@ -66,6 +70,7 @@ import org.apache.ignite.jiraservice.Ticket;
import org.apache.ignite.tcbot.common.conf.IGitHubConfig;
import org.apache.ignite.tcbot.common.conf.IJiraServerConfig;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
+import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.pr.BranchTicketMatcher;
import org.apache.ignite.tcbot.engine.pr.PrChainsProcessor;
@@ -95,6 +100,9 @@ public class TcBotTriggerAndSignOffService {
/** Logger. */
private static final Logger logger =
LoggerFactory.getLogger(TcBotTriggerAndSignOffService.class);
+ /** Slow visa operation threshold. */
+ private static final long SLOW_VISA_OPERATION_WARN_MS =
Long.getLong("tcbot.visa.slowOperationWarnMs", 1000L);
+
/** */
private static final ThreadLocal<DateFormat> THREAD_FORMATTER = new
ThreadLocal<DateFormat>() {
@Override protected DateFormat initialValue() {
@@ -255,6 +263,7 @@ public class TcBotTriggerAndSignOffService {
}
}
+ @AutoProfiling
@NotNull public String triggerBuildsAndObserve(
@Nullable String srvCodeOrAlias,
@Nullable String branchForTc,
@@ -267,15 +276,27 @@ public class TcBotTriggerAndSignOffService {
@Nullable String baseBranchForTc,
@Nonnull Boolean cleanRebuild,
@Nullable ITcBotUserCreds prov) {
+ long startNanos = System.nanoTime();
+ long initNanos = 0;
+ long prLookupNanos = 0;
+ long triggerNanos = 0;
+ long syncNanos = 0;
+ long observeNanos = 0;
+ int triggeredBuilds = 0;
+
String jiraRes = "";
+ long stepStart = System.nanoTime();
ITeamcityIgnited teamcity = tcIgnitedProv.server(srvCodeOrAlias, prov);
IGitHubConnIgnited ghIgn =
gitHubConnIgnitedProvider.server(srvCodeOrAlias);
+ initNanos = System.nanoTime() - stepStart;
if(!Strings.isNullOrEmpty(prNum)) {
try {
+ stepStart = System.nanoTime();
PullRequest pr = ghIgn.getPullRequest(Integer.parseInt(prNum));
+ prLookupNanos += System.nanoTime() - stepStart;
if(pr!=null) {
String shaShort = pr.lastCommitShaShort();
@@ -296,16 +317,34 @@ public class TcBotTriggerAndSignOffService {
Set<Integer> buildidsToSync = new HashSet<>();
for (int i = 0; i < suiteIds.length; i++) {
+ stepStart = System.nanoTime();
T2<Build, Set<Integer>> objects =
teamcity.triggerBuild(suiteIds[i], branchForTc, cleanRebuild, top != null &&
top, new HashMap<>(),
false, "");
+ triggerNanos += System.nanoTime() - stepStart;
+ triggeredBuilds++;
buildidsToSync.addAll(objects.get2());
builds[i] = objects.get1();
}
+ stepStart = System.nanoTime();
teamcity.fastBuildsSync(buildidsToSync);
+ syncNanos = System.nanoTime() - stepStart;
- if (observe != null && observe)
+ if (observe != null && observe) {
+ stepStart = System.nanoTime();
jiraRes += observeJira(srvCodeOrAlias, branchForTc, ticketId,
prov, parentSuiteId, baseBranchForTc, builds);
+ observeNanos = System.nanoTime() - stepStart;
+ }
+
+ long totalMs = millisSince(startNanos);
+ if (totalMs >= SLOW_VISA_OPERATION_WARN_MS) {
+ logger.warn("Slow visa trigger budget: server={}, branch={},
parentSuite={}, suites={}, totalMs={}, " +
+ "initMs={}, prLookupMs={}, triggerMs={}, fastSyncMs={},
observeMs={}, triggeredBuilds={}, " +
+ "buildIdsToSync={}",
+ srvCodeOrAlias, branchForTc, parentSuiteId, suiteIdList,
totalMs,
+ nanosToMillis(initNanos), nanosToMillis(prLookupNanos),
nanosToMillis(triggerNanos),
+ nanosToMillis(syncNanos), nanosToMillis(observeNanos),
triggeredBuilds, buildidsToSync.size());
+ }
return jiraRes;
}
@@ -399,28 +438,60 @@ public class TcBotTriggerAndSignOffService {
* @param srvCodeOrAlias Server id.
* @param credsProv Credentials
*/
+ @AutoProfiling
public List<ContributionToCheck> getContributionsToCheck(String
srvCodeOrAlias,
ITcBotUserCreds credsProv) {
+ long startNanos = System.nanoTime();
+ long serviceResolveNanos;
+ long prsLoadNanos;
+ long ticketsLoadNanos;
+ long defaultBuildTypeNanos;
+ long prLoopNanos;
+ AtomicLong prTicketResolveNanos = new AtomicLong();
+ AtomicLong prBuildLookupNanos = new AtomicLong();
+ long branchesLoadNanos;
+ long activeTicketsFilterNanos;
+ long activeTicketsLoopNanos;
+ AtomicLong activeBranchResolveNanos = new AtomicLong();
+ AtomicLong activeBranchCheckNanos = new AtomicLong();
+ AtomicLong activeBuildLookupNanos = new AtomicLong();
+ AtomicInteger prBuildLookupCnt = new AtomicInteger();
+ AtomicInteger activeBuildLookupCnt = new AtomicInteger();
+
+ long stepStart = System.nanoTime();
IJiraIgnited jiraIntegration = jiraIgnProv.server(srvCodeOrAlias);
IGitHubConnIgnited gitHubConnIgnited =
gitHubConnIgnitedProvider.server(srvCodeOrAlias);
ITeamcityIgnited tcIgn = tcIgnitedProv.server(srvCodeOrAlias,
credsProv);
+ serviceResolveNanos = System.nanoTime() - stepStart;
+ stepStart = System.nanoTime();
List<PullRequest> prs = gitHubConnIgnited.getPullRequests();
+ prsLoadNanos = System.nanoTime() - stepStart;
+ stepStart = System.nanoTime();
Set<Ticket> tickets = jiraIntegration.getTickets();
+ ticketsLoadNanos = System.nanoTime() - stepStart;
+
+ Map<String, Ticket> ticketsByKey = tickets.stream()
+ .filter(ticket -> ticket.key != null)
+ .collect(Collectors.toMap(ticket -> ticket.key, ticket -> ticket,
(first, second) -> first));
IJiraServerConfig jiraCfg = jiraIntegration.config();
IGitHubConfig ghCfg = gitHubConnIgnited.config();
+ stepStart = System.nanoTime();
String defBtForTcServ = findDefaultBuildType(srvCodeOrAlias);
+ defaultBuildTypeNanos = System.nanoTime() - stepStart;
List<ContributionToCheck> contribsList = new ArrayList<>();
+ stepStart = System.nanoTime();
if (prs != null) {
prs.forEach(pr -> {
ContributionToCheck c = new ContributionToCheck();
+ String prHeadRef = pr.head() == null ? null : pr.head().ref();
c.prNumber = pr.getNumber();
c.prTitle = pr.getTitle();
@@ -438,11 +509,16 @@ public class TcBotTriggerAndSignOffService {
c.prAuthorAvatarUrl = "";
}
- Ticket ticket =
ticketMatcher.resolveTicketIdForPrBasedContrib(tickets, jiraCfg, pr.getTitle());
+ long ticketStart = System.nanoTime();
+ Ticket ticket =
ticketMatcher.resolveTicketIdForPrBasedContrib(tickets, ticketsByKey, jiraCfg,
pr.getTitle());
+ prTicketResolveNanos.addAndGet(System.nanoTime() -
ticketStart);
if (ticket == null || ticket.id == 0) {
- if (pr.head() != null && pr.head().ref() != null)
- ticket =
ticketMatcher.resolveTicketIdForPrBasedContrib(tickets, jiraCfg,
pr.head().ref());
+ if (prHeadRef != null) {
+ ticketStart = System.nanoTime();
+ ticket =
ticketMatcher.resolveTicketIdForPrBasedContrib(tickets, ticketsByKey, jiraCfg,
prHeadRef);
+ prTicketResolveNanos.addAndGet(System.nanoTime() -
ticketStart);
+ }
}
c.jiraIssueId = ticket == null ? null : ticket.key;
@@ -452,34 +528,55 @@ public class TcBotTriggerAndSignOffService {
&& jiraCfg.getUrl() != null)
c.jiraIssueUrl =
jiraIntegration.generateTicketUrl(c.jiraIssueId);
- findBuildsForPr(defBtForTcServ,
Integer.toString(pr.getNumber()), gitHubConnIgnited, tcIgn)
+ long buildLookupStart = System.nanoTime();
+ findBuildsForPr(defBtForTcServ,
Integer.toString(pr.getNumber()), prHeadRef, gitHubConnIgnited, tcIgn)
.stream()
.map(buildRefCompacted ->
buildRefCompacted.branchName(compactor))
.findAny()
.ifPresent(bName -> c.tcBranchName = bName);
+ prBuildLookupNanos.addAndGet(System.nanoTime() -
buildLookupStart);
+ prBuildLookupCnt.incrementAndGet();
contribsList.add(c);
});
}
+ prLoopNanos = System.nanoTime() - stepStart;
+ stepStart = System.nanoTime();
List<String> branches = gitHubConnIgnited.getBranches();
+ Set<String> branchesSet = new HashSet<>(branches);
+ branchesLoadNanos = System.nanoTime() - stepStart;
+ stepStart = System.nanoTime();
List<Ticket> activeTickets = tickets.stream()
.filter(ticket ->
JiraTicketStatusCode.isActiveContribution(ticket.status()))
.collect(Collectors.toList());
+ activeTicketsFilterNanos = System.nanoTime() - stepStart;
+ stepStart = System.nanoTime();
activeTickets.forEach(ticket -> {
+ long branchResolveStart = System.nanoTime();
String branch = ticketMatcher.resolveTcBranchForPrLess(ticket,
jiraCfg,
ghCfg);
+ activeBranchResolveNanos.addAndGet(System.nanoTime() -
branchResolveStart);
if (Strings.isNullOrEmpty(branch))
return; // nothing to do if branch was not resolved
+ long branchCheckStart = System.nanoTime();
+ boolean branchExists = branchesSet.contains(branch);
+ activeBranchCheckNanos.addAndGet(System.nanoTime() -
branchCheckStart);
+
+ if (!branchExists) {
+ long buildLookupStart = System.nanoTime();
+ boolean buildsExist =
!tcIgn.getAllBuildsCompacted(defBtForTcServ, branch).isEmpty();
+ activeBuildLookupNanos.addAndGet(System.nanoTime() -
buildLookupStart);
+ activeBuildLookupCnt.incrementAndGet();
- if (!branches.contains(branch)
- && tcIgn.getAllBuildsCompacted(defBtForTcServ,
branch).isEmpty())
- return; //Skipping contributions without builds
+ if (!buildsExist)
+ return; //Skipping contributions without builds
+ }
ContributionToCheck contribution = new ContributionToCheck();
@@ -509,6 +606,25 @@ public class TcBotTriggerAndSignOffService {
contribsList.add(contribution);
});
+ activeTicketsLoopNanos = System.nanoTime() - stepStart;
+
+ long totalMs = millisSince(startNanos);
+ if (totalMs >= SLOW_VISA_OPERATION_WARN_MS) {
+ logger.warn("Slow visa contributions budget: server={},
totalMs={}, result={}, prs={}, tickets={}, " +
+ "activeTickets={}, branches={}, serviceResolveMs={},
prsLoadMs={}, ticketsLoadMs={}, " +
+ "defaultBuildTypeMs={}, prLoopMs={}, prTicketResolveMs={},
prBuildLookupMs={}, " +
+ "prBuildLookups={}, branchesLoadMs={}, activeFilterMs={},
activeLoopMs={}, " +
+ "activeBranchResolveMs={}, activeBranchCheckMs={},
activeBuildLookupMs={}, activeBuildLookups={}",
+ srvCodeOrAlias, totalMs, contribsList.size(), prs == null ? 0
: prs.size(), tickets.size(),
+ activeTickets.size(), branches.size(),
nanosToMillis(serviceResolveNanos), nanosToMillis(prsLoadNanos),
+ nanosToMillis(ticketsLoadNanos),
nanosToMillis(defaultBuildTypeNanos), nanosToMillis(prLoopNanos),
+ nanosToMillis(prTicketResolveNanos.get()),
nanosToMillis(prBuildLookupNanos.get()),
+ prBuildLookupCnt.get(),
+ nanosToMillis(branchesLoadNanos),
nanosToMillis(activeTicketsFilterNanos),
+ nanosToMillis(activeTicketsLoopNanos),
nanosToMillis(activeBranchResolveNanos.get()),
+ nanosToMillis(activeBranchCheckNanos.get()),
nanosToMillis(activeBuildLookupNanos.get()),
+ activeBuildLookupCnt.get());
+ }
return contribsList;
}
@@ -524,9 +640,25 @@ public class TcBotTriggerAndSignOffService {
String prId,
IGitHubConnIgnited ghConn,
ITeamcityIgnited srv) {
+ return findBuildsForPr(suiteId, prId, null, ghConn, srv);
+ }
+
+ /**
+ * @param suiteId Suite id.
+ * @param prId Pr id from {@link ContributionToCheck#prNumber}. Negative
value imples branch number for PR-less.
+ * @param prHeadBranch PR head branch name, when it is already known by
caller.
+ * @param ghConn Gh connection.
+ * @param srv TC Server connection.
+ */
+ @Nonnull
+ private List<BuildRefCompacted> findBuildsForPr(String suiteId,
+ String prId,
+ @Nullable String prHeadBranch,
+ IGitHubConnIgnited ghConn,
+ ITeamcityIgnited srv) {
List<BuildRefCompacted> buildHist = srv.getAllBuildsCompacted(suiteId,
- branchForTcDefault(prId, ghConn));
+ branchForTcDefault(prId, ghConn, prHeadBranch));
if (!buildHist.isEmpty())
return buildHist;
@@ -543,7 +675,7 @@ public class TcBotTriggerAndSignOffService {
String bracnhToCheck =
ghConn.config().isPreferBranches()
? branchForTcA(prId) // for prefer branches mode it
was already checked in default
- : getPrBranch(ghConn, prNum);
+ : prHeadBranch != null ? prHeadBranch :
getPrBranch(ghConn, prNum);
if (bracnhToCheck == null)
return Collections.emptyList();
@@ -574,12 +706,22 @@ public class TcBotTriggerAndSignOffService {
* @param ghConn Github integration.
*/
private String branchForTcDefault(String prId, IGitHubConnIgnited ghConn) {
+ return branchForTcDefault(prId, ghConn, null);
+ }
+
+ /**
+ * @param prId Pr id from {@link ContributionToCheck#prNumber}. Negative
value imples branch number to be used for
+ * PR-less contributions.
+ * @param ghConn Github integration.
+ * @param prHeadBranch PR head branch name, when it is already known by
caller.
+ */
+ private String branchForTcDefault(String prId, IGitHubConnIgnited ghConn,
@Nullable String prHeadBranch) {
Integer prNum = Integer.valueOf(prId);
if (prNum < 0)
return ghConn.gitBranchPrefix() + (-prNum); // Checking
"ignite-10930" builds only
if (ghConn.config().isPreferBranches()) {
- String ref = getPrBranch(ghConn, prNum);
+ String ref = prHeadBranch != null ? prHeadBranch :
getPrBranch(ghConn, prNum);
if (ref != null)
return ref;
}
@@ -601,26 +743,46 @@ public class TcBotTriggerAndSignOffService {
* @param prId Pr id from {@link ContributionToCheck#prNumber}. Negative
value imples branch number (with
* appropriate prefix from GH config).
*/
+ @AutoProfiling
public Set<ContributionCheckStatus> contributionStatuses(String
srvCodeOrAlias, ITcBotUserCreds prov,
String prId) {
+ long startNanos = System.nanoTime();
+ long serviceResolveNanos;
+ long defaultBuildTypeNanos;
+ long buildTypesNanos;
+ long buildLookupNanos = 0;
+ long statusBuildNanos = 0;
+ int buildLookupCnt = 0;
+
Set<ContributionCheckStatus> statuses = new LinkedHashSet<>();
+ long stepStart = System.nanoTime();
ITeamcityIgnited teamcity = tcIgnitedProv.server(srvCodeOrAlias, prov);
- String defaultBuildType = findDefaultBuildType(srvCodeOrAlias);
-
IGitHubConnIgnited ghConn =
gitHubConnIgnitedProvider.server(srvCodeOrAlias);
+ serviceResolveNanos = System.nanoTime() - stepStart;
+
+ stepStart = System.nanoTime();
+ String defaultBuildType = findDefaultBuildType(srvCodeOrAlias);
+ defaultBuildTypeNanos = System.nanoTime() - stepStart;
Preconditions.checkState(ghConn.config().code().equals(srvCodeOrAlias));
+ stepStart = System.nanoTime();
List<String> compositeBuildTypeIds =
findApplicableBuildTypes(srvCodeOrAlias, teamcity);
+ buildTypesNanos = System.nanoTime() - stepStart;
for (String btId : compositeBuildTypeIds) {
+ stepStart = System.nanoTime();
List<BuildRefCompacted> buildsForBt = findBuildsForPr(btId, prId,
ghConn, teamcity);
+ buildLookupNanos += System.nanoTime() - stepStart;
+ buildLookupCnt++;
+ stepStart = System.nanoTime();
ContributionCheckStatus contributionAgainstSuite =
buildsForBt.isEmpty()
? new ContributionCheckStatus(btId, branchForTcDefault(prId,
ghConn))
: contributionStatus(srvCodeOrAlias, btId, buildsForBt,
teamcity, ghConn, prId);
+ statusBuildNanos += System.nanoTime() - stepStart;
if(Objects.equals(btId, defaultBuildType))
contributionAgainstSuite.defaultBuildType = true;
@@ -628,6 +790,16 @@ public class TcBotTriggerAndSignOffService {
statuses.add(contributionAgainstSuite);
}
+ long totalMs = millisSince(startNanos);
+ if (totalMs >= SLOW_VISA_OPERATION_WARN_MS) {
+ logger.warn("Slow visa contributionStatus budget: server={},
prId={}, totalMs={}, statuses={}, " +
+ "serviceResolveMs={}, defaultBuildTypeMs={},
buildTypesMs={}, buildLookupMs={}, " +
+ "statusBuildMs={}, buildLookups={}",
+ srvCodeOrAlias, prId, totalMs, statuses.size(),
nanosToMillis(serviceResolveNanos),
+ nanosToMillis(defaultBuildTypeNanos),
nanosToMillis(buildTypesNanos), nanosToMillis(buildLookupNanos),
+ nanosToMillis(statusBuildNanos), buildLookupCnt);
+ }
+
return statuses;
}
@@ -768,18 +940,25 @@ public class TcBotTriggerAndSignOffService {
return teamcity.host() + "viewQueued.html?itemId=" + ref.id();
}
+ @AutoProfiling
public CurrentVisaStatus currentVisaStatus(String srvCode, ITcBotUserCreds
prov, String buildTypeId,
String tcBranch) {
+ long startNanos = System.nanoTime();
CurrentVisaStatus status = new CurrentVisaStatus();
List<ShortSuiteUi> suitesStatuses
= prChainsProcessor.getBlockersSuitesStatuses(buildTypeId,
tcBranch, srvCode, prov, SyncMode.NONE, null);
- if (suitesStatuses == null)
+ if (suitesStatuses == null) {
+ logSlowVisaOperation(startNanos, "visaStatus", srvCode,
buildTypeId, tcBranch, 0);
+
return status;
+ }
status.blockers =
suitesStatuses.stream().mapToInt(ShortSuiteUi::totalBlockers).sum();
+ logSlowVisaOperation(startNanos, "visaStatus", srvCode, buildTypeId,
tcBranch, suitesStatuses.size());
+
return status;
}
@@ -795,6 +974,7 @@ public class TcBotTriggerAndSignOffService {
* @param baseBranchForTc Base branch in TC identification
* @return {@link Visa} instance.
*/
+ @AutoProfiling
public Visa notifyJira(
String srvCodeOrAlias,
ITcBotUserCreds prov,
@@ -802,14 +982,18 @@ public class TcBotTriggerAndSignOffService {
String branchForTc,
String ticket,
@Nullable String baseBranchForTc) {
+ long startNanos = System.nanoTime();
ITeamcityIgnited tcIgnited = tcIgnitedProv.server(srvCodeOrAlias,
prov);
IJiraIgnited jira = jiraIgnProv.server(srvCodeOrAlias);
List<Integer> builds =
tcIgnited.getLastNBuildsFromHistory(buildTypeId, branchForTc, 1);
- if (builds.isEmpty())
+ if (builds.isEmpty()) {
+ logSlowVisaOperation(startNanos, "notifyJiraNoBuilds",
srvCodeOrAlias, buildTypeId, branchForTc, 0);
+
return new Visa("JIRA wasn't commented - no finished builds to
analyze.");
+ }
Integer buildId = builds.get(0);
@@ -852,6 +1036,35 @@ public class TcBotTriggerAndSignOffService {
return new Visa("JIRA wasn't commented - " + errMsg);
}
+ logSlowVisaOperation(startNanos, "notifyJira", srvCodeOrAlias,
buildTypeId, branchForTc, blockers);
+
return new Visa(Visa.JIRA_COMMENTED, res, blockers);
}
+
+ /**
+ * @param nanos Nanoseconds.
+ */
+ private static long nanosToMillis(long nanos) {
+ return TimeUnit.NANOSECONDS.toMillis(nanos);
+ }
+
+ /**
+ * @param startNanos Start time.
+ */
+ private static long millisSince(long startNanos) {
+ return nanosToMillis(System.nanoTime() - startNanos);
+ }
+
+ /**
+ * Logs slow visa operation.
+ */
+ private void logSlowVisaOperation(long startNanos, String op, String
srvCode, String buildTypeId, String tcBranch,
+ int resultSize) {
+ long totalMs = millisSince(startNanos);
+
+ if (totalMs >= SLOW_VISA_OPERATION_WARN_MS) {
+ logger.warn("Slow visa operation: op={}, server={}, buildType={},
branch={}, totalMs={}, resultSize={}",
+ op, srvCode, buildTypeId, tcBranch, totalMs, resultSize);
+ }
+ }
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/TcApplicationResCfg.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/TcApplicationResCfg.java
index 84316689..951ce2a2 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/TcApplicationResCfg.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/TcApplicationResCfg.java
@@ -20,6 +20,7 @@ package org.apache.ignite.ci.web;
import org.apache.ignite.ci.web.auth.AuthenticationFilter;
import org.apache.ignite.ci.web.rest.exception.ExeptionsTraceLogger;
import org.apache.ignite.ci.web.rest.exception.ServiceStartingExceptionMapper;
+import org.apache.ignite.ci.web.rest.monitoring.RestRequestTimingFilter;
import org.apache.ignite.tcbot.common.exeption.ServiceUnauthorizedException;
import org.glassfish.jersey.logging.LoggingFeature;
import org.glassfish.jersey.server.ResourceConfig;
@@ -40,5 +41,6 @@ public class TcApplicationResCfg extends ResourceConfig {
register(LoggingFeature.class);
register(ExeptionsTraceLogger.class);
+ register(RestRequestTimingFilter.class);
}
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
index c0d1c9b7..65dc2a0a 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java
@@ -480,4 +480,24 @@ public class MonitoringService {
}
return res;
}
+
+ @GET
+ @Path("requests")
+ public List<RequestStat> getRequestStats() {
+ return RestRequestTimingStorage.stats();
+ }
+
+ @GET
+ @Path("recentRequests")
+ public List<RequestTiming> getRecentRequests() {
+ return RestRequestTimingStorage.recent();
+ }
+
+ @POST
+ @Path("resetRequests")
+ public SimpleResult resetRequestStats() {
+ RestRequestTimingStorage.reset();
+
+ return new SimpleResult("Ok");
+ }
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/RequestStat.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/RequestStat.java
new file mode 100644
index 00000000..9f2e4a37
--- /dev/null
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/RequestStat.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.web.rest.monitoring;
+
+/**
+ * Aggregated REST request timing.
+ */
+@SuppressWarnings("WeakerAccess")
+public class RequestStat {
+ public String method;
+ public String path;
+ public long count;
+ public long totalMs;
+ public long avgMs;
+ public long maxMs;
+ public long lastMs;
+ public int lastStatus;
+ public String lastRequest;
+}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/TcApplicationResCfg.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/RequestTiming.java
similarity index 50%
copy from
ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/TcApplicationResCfg.java
copy to
ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/RequestTiming.java
index 84316689..74fa858e 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/TcApplicationResCfg.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/RequestTiming.java
@@ -14,31 +14,37 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.ignite.ci.web;
-
-import org.apache.ignite.ci.web.auth.AuthenticationFilter;
-import org.apache.ignite.ci.web.rest.exception.ExeptionsTraceLogger;
-import org.apache.ignite.ci.web.rest.exception.ServiceStartingExceptionMapper;
-import org.apache.ignite.tcbot.common.exeption.ServiceUnauthorizedException;
-import org.glassfish.jersey.logging.LoggingFeature;
-import org.glassfish.jersey.server.ResourceConfig;
+package org.apache.ignite.ci.web.rest.monitoring;
/**
- * Resource config for Jersey Application
+ * Single REST request timing.
*/
-public class TcApplicationResCfg extends ResourceConfig {
+@SuppressWarnings("WeakerAccess")
+public class RequestTiming {
+ public String method;
+ public String path;
+ public int status;
+ public long durationMs;
+ public long timestamp;
+
/**
* Default constructor.
*/
- public TcApplicationResCfg() {
- //Register Auth Filter here
- register(AuthenticationFilter.class);
-
- register(ServiceUnauthorizedException.class);
- register(ServiceStartingExceptionMapper.class);
+ public RequestTiming() {
+ }
- register(LoggingFeature.class);
- register(ExeptionsTraceLogger.class);
+ /**
+ * @param method HTTP method.
+ * @param path Request path with query.
+ * @param status Response status.
+ * @param durationMs Request duration in millis.
+ * @param timestamp Completion timestamp.
+ */
+ public RequestTiming(String method, String path, int status, long
durationMs, long timestamp) {
+ this.method = method;
+ this.path = path;
+ this.status = status;
+ this.durationMs = durationMs;
+ this.timestamp = timestamp;
}
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/RestRequestTimingFilter.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/RestRequestTimingFilter.java
new file mode 100644
index 00000000..a32799bc
--- /dev/null
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/RestRequestTimingFilter.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.web.rest.monitoring;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Priority;
+import javax.ws.rs.Priorities;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ContainerResponseContext;
+import javax.ws.rs.container.ContainerResponseFilter;
+import javax.ws.rs.ext.Provider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Logs slow REST requests with method, path and response status.
+ */
+@Provider
+@Priority(Priorities.USER)
+public class RestRequestTimingFilter implements ContainerRequestFilter,
ContainerResponseFilter {
+ /** Logger. */
+ private static final Logger logger =
LoggerFactory.getLogger(RestRequestTimingFilter.class);
+
+ /** Request context property with request start time. */
+ private static final String START_TIME_NANOS =
RestRequestTimingFilter.class.getName() + ".startTimeNanos";
+
+ /** Slow REST request threshold. */
+ private static final long SLOW_REQUEST_WARN_MS =
Long.getLong("tcbot.rest.slowRequestWarnMs", 1000L);
+
+ /** {@inheritDoc} */
+ @Override public void filter(ContainerRequestContext reqCtx) throws
IOException {
+ reqCtx.setProperty(START_TIME_NANOS, System.nanoTime());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void filter(ContainerRequestContext reqCtx,
ContainerResponseContext respCtx) throws IOException {
+ Object startObj = reqCtx.getProperty(START_TIME_NANOS);
+
+ if (!(startObj instanceof Long))
+ return;
+
+ long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() -
(Long)startObj);
+
+ String rawQry = reqCtx.getUriInfo().getRequestUri().getRawQuery();
+ String path = reqCtx.getUriInfo().getPath();
+ String pathWithQuery = path;
+
+ if (rawQry != null)
+ pathWithQuery += "?" + rawQry;
+
+ RestRequestTimingStorage.record(reqCtx.getMethod(), path,
pathWithQuery, respCtx.getStatus(), elapsedMs);
+
+ if (elapsedMs < SLOW_REQUEST_WARN_MS)
+ return;
+
+ logger.warn("Slow REST request: method={}, path={}, status={},
durationMs={}",
+ reqCtx.getMethod(), pathWithQuery, respCtx.getStatus(), elapsedMs);
+ }
+}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/RestRequestTimingStorage.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/RestRequestTimingStorage.java
new file mode 100644
index 00000000..3e623779
--- /dev/null
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/RestRequestTimingStorage.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ci.web.rest.monitoring;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * In-memory REST request timing storage.
+ */
+public class RestRequestTimingStorage {
+ /** Max recent requests. */
+ private static final int MAX_RECENT =
Integer.getInteger("tcbot.rest.recentRequestsSize", 200);
+
+ /** Aggregated stats by method and path. */
+ private static final ConcurrentMap<String, StatAccumulator> STATS = new
ConcurrentHashMap<>();
+
+ /** Recent completed requests. */
+ private static final ConcurrentLinkedDeque<RequestTiming> RECENT = new
ConcurrentLinkedDeque<>();
+
+ /**
+ * @param method HTTP method.
+ * @param path Path without query.
+ * @param pathWithQuery Path with query.
+ * @param status Response status.
+ * @param durationMs Duration in millis.
+ */
+ public static void record(String method, String path, String
pathWithQuery, int status, long durationMs) {
+ RequestTiming timing = new RequestTiming(method, pathWithQuery,
status, durationMs, System.currentTimeMillis());
+
+ String key = method + " " + path;
+ STATS.computeIfAbsent(key, k -> new StatAccumulator(method,
path)).record(timing);
+
+ RECENT.addFirst(timing);
+
+ while (RECENT.size() > MAX_RECENT)
+ RECENT.pollLast();
+ }
+
+ /**
+ * @return Aggregated stats.
+ */
+ public static List<RequestStat> stats() {
+ List<RequestStat> res = new ArrayList<>();
+
+ STATS.values().forEach(stat -> res.add(stat.snapshot()));
+
+ res.sort(Comparator.comparing((RequestStat stat) ->
stat.totalMs).reversed());
+
+ return res;
+ }
+
+ /**
+ * @return Recent request timings.
+ */
+ public static List<RequestTiming> recent() {
+ return new ArrayList<>(RECENT);
+ }
+
+ /**
+ * Clears accumulated stats.
+ */
+ public static void reset() {
+ STATS.clear();
+ RECENT.clear();
+ }
+
+ /**
+ * Mutable request stats accumulator.
+ */
+ private static class StatAccumulator {
+ /** HTTP method. */
+ private final String method;
+
+ /** Path without query. */
+ private final String path;
+
+ /** Count. */
+ private final AtomicLong count = new AtomicLong();
+
+ /** Total duration. */
+ private final AtomicLong totalMs = new AtomicLong();
+
+ /** Max duration. */
+ private final AtomicLong maxMs = new AtomicLong();
+
+ /** Last duration. */
+ private final AtomicLong lastMs = new AtomicLong();
+
+ /** Last status. */
+ private final AtomicInteger lastStatus = new AtomicInteger();
+
+ /** Last request path with query. */
+ private final AtomicReference<String> lastRequest = new
AtomicReference<>();
+
+ /**
+ * @param method HTTP method.
+ * @param path Path without query.
+ */
+ private StatAccumulator(String method, String path) {
+ this.method = method;
+ this.path = path;
+ }
+
+ /**
+ * @param timing Timing.
+ */
+ private void record(RequestTiming timing) {
+ count.incrementAndGet();
+ totalMs.addAndGet(timing.durationMs);
+ maxMs.accumulateAndGet(timing.durationMs, Math::max);
+ lastMs.set(timing.durationMs);
+ lastStatus.set(timing.status);
+ lastRequest.set(timing.path);
+ }
+
+ /**
+ * @return Snapshot.
+ */
+ private RequestStat snapshot() {
+ RequestStat stat = new RequestStat();
+ stat.method = method;
+ stat.path = path;
+ stat.count = count.get();
+ stat.totalMs = totalMs.get();
+ stat.avgMs = stat.count == 0 ? 0 : stat.totalMs / stat.count;
+ stat.maxMs = maxMs.get();
+ stat.lastMs = lastMs.get();
+ stat.lastStatus = lastStatus.get();
+ stat.lastRequest = lastRequest.get();
+
+ return stat;
+ }
+ }
+}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java
index f5cd3a45..b0b969a3 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.servlet.ServletContext;
@@ -49,15 +50,24 @@ import org.apache.ignite.tcignited.SyncMode;
import org.apache.ignite.tcservice.model.mute.MuteInfo;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static
org.apache.ignite.tcignited.TeamcityIgnitedImpl.DEFAULT_PROJECT_ID;
@Path(GetTrackedBranchTestResults.TRACKED)
@Produces(MediaType.APPLICATION_JSON)
public class GetTrackedBranchTestResults {
+ /** Logger. */
+ private static final Logger logger =
LoggerFactory.getLogger(GetTrackedBranchTestResults.class);
+
public static final String TRACKED = "tracked";
public static final int DEFAULT_COUNT = 10;
+ /** Slow tracked branch request threshold. */
+ private static final long SLOW_TRACKED_RESULTS_WARN_MS =
+ Long.getLong("tcbot.tracked.slowResultsWarnMs", 1000L);
+
/** Servlet Context. */
@Context
private ServletContext ctx;
@@ -146,6 +156,8 @@ public class GetTrackedBranchTestResults {
@Nullable Integer showTestLongerThan,
@Nullable Boolean showMuted,
@Nullable Boolean showIgnored) {
+ long startNanos = System.nanoTime();
+
ITcBotUserCreds creds = ITcBotUserCreds.get(req);
Injector injector = CtxListener.getInjector(ctx);
@@ -154,7 +166,7 @@ public class GetTrackedBranchTestResults {
int maxDurationSec = (showTestLongerThan == null || showTestLongerThan
< 1) ? 0 : showTestLongerThan;
- return injector.getInstance(IDetailedStatusForTrackedBranch.class)
+ DsSummaryUi res =
injector.getInstance(IDetailedStatusForTrackedBranch.class)
.getTrackedBranchTestFailures(branch,
checkAllLogs,
actualMergeBuilds,
@@ -168,6 +180,10 @@ public class GetTrackedBranchTestResults {
maxDurationSec,
Boolean.TRUE.equals(showMuted),
Boolean.TRUE.equals(showIgnored));
+
+ logSlowTrackedResult(startNanos, "latest", branch, actualMergeBuilds,
mode, res);
+
+ return res;
}
@GET
@@ -201,14 +217,33 @@ public class GetTrackedBranchTestResults {
@QueryParam("count") Integer cnt,
@QueryParam("checkAllLogs") @Nullable Boolean checkAllLogs,
SyncMode mode) {
+ long startNanos = System.nanoTime();
+
ITcBotUserCreds creds = ITcBotUserCreds.get(req);
int cntLimit = cnt == null ? DEFAULT_COUNT : cnt;
Injector injector = CtxListener.getInjector(ctx);
- return injector.getInstance(TrackedBranchChainsProcessor.class)
+ DsSummaryUi res =
injector.getInstance(TrackedBranchChainsProcessor.class)
.getTrackedBranchTestFailures(branchOpt, checkAllLogs, cntLimit,
creds, mode,
false, null, null, DisplayMode.OnlyFailures, null,
-1, false, false);
+
+ logSlowTrackedResult(startNanos, "merged", branchOpt, cntLimit, mode,
res);
+
+ return res;
+ }
+
+ /**
+ * Logs slow tracked branch request.
+ */
+ private void logSlowTrackedResult(long startNanos, String op, @Nullable
String branch, int cnt, SyncMode mode,
+ DsSummaryUi res) {
+ long totalMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() -
startNanos);
+
+ if (totalMs >= SLOW_TRACKED_RESULTS_WARN_MS) {
+ logger.warn("Slow tracked branch result: op={}, branch={},
count={}, mode={}, totalMs={}, chains={}",
+ op, branch, cnt, mode, totalMs, res == null ? 0 :
res.servers.size());
+ }
}
/**
diff --git a/ignite-tc-helper-web/src/main/webapp/current.html
b/ignite-tc-helper-web/src/main/webapp/current.html
index e8c420c1..decd43c6 100644
--- a/ignite-tc-helper-web/src/main/webapp/current.html
+++ b/ignite-tc-helper-web/src/main/webapp/current.html
@@ -27,6 +27,9 @@
<script>
let g_shownDataHashCodeHex = "";
let g_checkForUpdateSched = false;
+ let g_fullLoadInProgress = false;
+ let g_partialLoadInProgress = false;
+ let g_partialDataShown = false;
let gVue;
function genLink() {
@@ -242,6 +245,9 @@ function loadData() {
var curFailuresUrl = "rest/tracked/results" + parmsForRest();
$("#loadStatus").html("⌛ Please wait");
+ g_fullLoadInProgress = true;
+ g_partialDataShown = false;
+
setTimeout(loadPartialData, 3000);
$.ajax({
@@ -256,6 +262,9 @@ function loadData() {
setTimeout(checkForUpdate, 20000);
}
},
+ complete: function() {
+ g_fullLoadInProgress = false;
+ },
error: showErrInLoadStatus
});
}
@@ -263,9 +272,12 @@ function loadData() {
function loadPartialData() {
var curFailuresUrl = "rest/tracked/resultsNoSync" + parmsForRest();
- if (g_shownDataHashCodeHex !== "") {
+ if (g_shownDataHashCodeHex !== "" || g_partialLoadInProgress ||
g_partialDataShown || !g_fullLoadInProgress) {
return;
}
+
+ g_partialLoadInProgress = true;
+
$.ajax({
url: curFailuresUrl,
success: function (result) {
@@ -283,9 +295,16 @@ function loadPartialData() {
}
}
if (validResult)
+ {
showData(result);
+ g_partialDataShown = true;
+ }
- setTimeout(loadPartialData, 3000);
+ if (!g_partialDataShown && g_fullLoadInProgress)
+ setTimeout(loadPartialData, 10000);
+ },
+ complete: function() {
+ g_partialLoadInProgress = false;
},
error: showErrInLoadStatus
});
@@ -451,4 +470,4 @@ function showData(result) {
<div id="version"></div>
<div style="visibility:hidden;"><div id="triggerConfirm" title="Trigger
Confirmation"></div><div id="triggerDialog" title="Trigger Result"></div></div>
</body>
-</html>
\ No newline at end of file
+</html>
diff --git a/ignite-tc-helper-web/src/main/webapp/monitoring.html
b/ignite-tc-helper-web/src/main/webapp/monitoring.html
index 0cf0703c..d9932882 100644
--- a/ignite-tc-helper-web/src/main/webapp/monitoring.html
+++ b/ignite-tc-helper-web/src/main/webapp/monitoring.html
@@ -36,6 +36,18 @@
});
}
+ function loadRequestData() {
+ $.ajax({
+ url: "rest/monitoring/requests",
+ success: function (result) {
+ $("#loadStatus").html("");
+
+ showRequests(result);
+ },
+ error: showErrInLoadStatus
+ });
+ }
+
function loadData() {
$("#loadStatus").html("⌛ Please wait");
$.ajax({
@@ -61,6 +73,7 @@
});
loadPofilingData();
+ loadRequestData();
$.ajax({
url: "rest/monitoring/cacheMetrics",
@@ -137,6 +150,40 @@
$("#profiling").html(res);
}
+ /**
+ * @param result org.apache.ignite.ci.web.rest.monitoring.RequestStat
+ */
+ function showRequests(result) {
+ var res = "<table class='stat'>" ;
+ res += "<tr>";
+ res += "<th>Method</th>";
+ res += "<th>Path</th>";
+ res += "<th>Count</th>";
+ res += "<th>Total, ms</th>";
+ res += "<th>Avg, ms</th>";
+ res += "<th>Max, ms</th>";
+ res += "<th>Last, ms</th>";
+ res += "<th>Last status</th>";
+ res += "<th>Last request</th>";
+ res += "</tr>";
+ for (var i = 0; i < result.length; i++) {
+ var inv = result[i];
+ res += "<tr>";
+ res += "<td>" + escapeHtml(inv.method) + "</td>";
+ res += "<td>" + escapeHtml(inv.path) + "</td>";
+ res += "<td>" + escapeHtml(inv.count) + "</td>";
+ res += "<td>" + escapeHtml(inv.totalMs) + "</td>";
+ res += "<td>" + escapeHtml(inv.avgMs) + "</td>";
+ res += "<td>" + escapeHtml(inv.maxMs) + "</td>";
+ res += "<td>" + escapeHtml(inv.lastMs) + "</td>";
+ res += "<td>" + escapeHtml(inv.lastStatus) + "</td>";
+ res += "<td>" + escapeHtml(inv.lastRequest) + "</td>";
+ res += "</tr>";
+ }
+ res += "</table>";
+ $("#requests").html(res);
+ }
+
function showCacheMetrics(result) {
var res = "<table class='stat'>" ;
res += "<tr>";
@@ -166,6 +213,17 @@
});
}
+ function resetRequests() {
+ $.ajax({
+ url: "rest/monitoring/resetRequests",
+ method: "post",
+ success: function(result) {
+ loadRequestData();
+ },
+ error: showErrInLoadStatus
+ });
+ }
+
function testSlackNotification() {
$.ajax({
url: "rest/monitoring/testSlackNotification",
@@ -198,7 +256,7 @@
}
function escapeHtml(str) {
- return String(str || "")
+ return String(str == null ? "" : str)
.replace(/&/g, "&")
.replace(/</g, "<")
.replace(/>/g, ">")
@@ -213,6 +271,11 @@ Tasks Monitoring Data:
<div id="tasks" style="font-family: monospace"></div>
<br>
+<hr>
+<b>REST Request Timings:</b> <button onclick="resetRequests()">Reset</button>
+<div id="requests" style="font-family: monospace"></div>
+<br>
+
<hr>
<b>Method Profiling Data:</b> <button onclick="resetProfiling()">Reset</button>
<div id="profiling" style="font-family: monospace"></div>
diff --git
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringServiceSecurityTest.java
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringServiceSecurityTest.java
new file mode 100644
index 00000000..6eae4b1e
--- /dev/null
+++
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringServiceSecurityTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.web.rest.monitoring;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import javax.annotation.security.PermitAll;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class MonitoringServiceSecurityTest {
+ @Test
+ public void requestTimingEndpointsRequireAuthentication() throws
NoSuchMethodException {
+
assertAuthRequired(MonitoringService.class.getMethod("getRequestStats"));
+
assertAuthRequired(MonitoringService.class.getMethod("getRecentRequests"));
+
assertAuthRequired(MonitoringService.class.getMethod("resetRequestStats"));
+ }
+
+ @Test
+ public void requestTimingFieldsAreEscaped() throws IOException {
+ String html = new String(Files.readAllBytes(monitoringHtml()),
StandardCharsets.UTF_8);
+
+ assertTrue(html.contains("escapeHtml(inv.method)"));
+ assertTrue(html.contains("escapeHtml(inv.path)"));
+ assertTrue(html.contains("escapeHtml(inv.lastRequest)"));
+ assertTrue(html.contains("String(str == null ? \"\" : str)"));
+ }
+
+ private static void assertAuthRequired(Method method) {
+ assertFalse(method.isAnnotationPresent(PermitAll.class));
+ }
+
+ private static Path monitoringHtml() {
+ Path projectPath = Paths.get("src/main/webapp/monitoring.html");
+
+ if (Files.exists(projectPath))
+ return projectPath;
+
+ return
Paths.get("ignite-tc-helper-web/src/main/webapp/monitoring.html");
+ }
+}
diff --git a/migrator/build.gradle b/migrator/build.gradle
index bd541325..d47fd114 100644
--- a/migrator/build.gradle
+++ b/migrator/build.gradle
@@ -10,8 +10,25 @@ dependencies {
implementation "org.apache.ignite:ignite-core:$ignVer"
implementation 'ch.qos.logback:logback-classic:1.2.3'
implementation project(':tcbot-common')
+
+ testImplementation project(':ignite-tc-helper-web')
+ testImplementation project(':tcbot-persistence')
+ testImplementation project(':tcbot-teamcity-ignited')
+ testImplementation "org.apache.ignite:ignite-slf4j:$ignVer"
+ testImplementation group: 'junit', name: 'junit', version: junitVer
}
application {
mainClass = 'org.apache.ignite.migrate.GridIntListMigrator'
}
+
+test {
+ jvmArgs igniteJava17JvmArgs
+ maxHeapSize = '1536m'
+ systemProperty 'compat.work.dir',
file('src/test/work/ignite-db-compat').absolutePath
+
+ testLogging {
+ events 'passed', 'failed', 'skipped', 'standardOut', 'standardError'
+ showStandardStreams = true
+ }
+}
diff --git
a/migrator/src/test/java/org/apache/ignite/migrate/LegacyPersistentStorageCompatibilityTest.java
b/migrator/src/test/java/org/apache/ignite/migrate/LegacyPersistentStorageCompatibilityTest.java
new file mode 100644
index 00000000..6dcd5d17
--- /dev/null
+++
b/migrator/src/test/java/org/apache/ignite/migrate/LegacyPersistentStorageCompatibilityTest.java
@@ -0,0 +1,1120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.migrate;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.ci.db.DbMigrations;
+import org.apache.ignite.ci.db.Ignite2Configurer;
+import org.apache.ignite.ci.db.TcHelperDb;
+import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
+import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureHandler;
+import org.apache.ignite.logger.slf4j.Slf4jLogger;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.tcbot.common.util.GridIntList;
+import org.apache.ignite.tcbot.persistence.CacheConfigs;
+import org.apache.ignite.tcbot.persistence.IStringCompactor;
+import org.apache.ignite.tcignited.ITeamcityIgnited;
+import org.apache.ignite.tcignited.build.FatBuildDao;
+import org.apache.ignite.tcignited.buildref.BuildRefDao;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static
org.apache.ignite.tcbot.persistence.IgniteStringCompactor.STRINGS_CACHE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Covers opening and migrating a TC Bot persistent storage created by old bot
sources on Ignite 2.14.
+ */
+public class LegacyPersistentStorageCompatibilityTest {
+ /** Megabyte in bytes. */
+ private static final long MB = 1024L * 1024L;
+
+ /** Last commit before IGNITE-27101. Its build uses Ignite 2.14.0 and Java
11 toolchain. */
+ private static final String LEGACY_COMMIT = "7437ec80^";
+
+ /** Expected Ignite version in the old checkout. */
+ private static final String LEGACY_IGNITE_VERSION = "2.14.0";
+
+ /** Ignite 2.14 default WAL segments count used by the old bot
configuration. */
+ private static final int LEGACY_WAL_SEGMENTS = 10;
+
+ /** Ignite 2.14 default WAL segment size used by the old bot
configuration. */
+ private static final long LEGACY_WAL_SEGMENT_SIZE = 64 * MB;
+
+ /** Test TeamCity server id. Keep it different from apache to avoid
collisions with real local work dirs. */
+ private static final String SRV_ID = "perf-test-tc";
+
+ /** Project/suite that represents Ignite on Java 8 in TeamCity. */
+ private static final String RUN_ALL_JAVA_8 = "IgniteTests24Java8_RunAll";
+
+ /** Branch used by the local perf compatibility test. */
+ private static final String BRANCH = "refs/heads/perf-test-master";
+
+ /** Cache that intentionally stores Ignite 2.14 internal GridIntList for
migrator coverage. */
+ private static final String LEGACY_GRID_INT_LIST_CACHE =
"legacyGridIntListCache";
+
+ /** Cache used only to produce enough WAL with old Ignite. */
+ private static final String LEGACY_WAL_STRESS_CACHE =
"legacyWalStressCache";
+
+ /** Small durable region for the test. */
+ private static final long REGION_SIZE = 256L * 1024 * 1024;
+
+ /** */
+ @Rule public TemporaryFolder tmp = new TemporaryFolder();
+
+ /**
+ * Creates a small durable store with old bot code and Ignite 2.14, then
opens and migrates it with the current code.
+ */
+ @Test
+ public void currentNodeStartsAndMigratesDatabaseCreatedByLegacyBot()
throws Exception {
+ File legacyCheckout = tmp.newFolder("legacy-checkout");
+ File workRoot = compatibilityWorkRoot();
+ File workDir = new File(workRoot, "legacy-tcbot-work");
+
+ recreateDirectory(workRoot, workDir);
+
+ System.out.println("Compat DB work root: " +
workRoot.getAbsolutePath());
+ System.out.println("Compat DB left after test at: " +
workDir.getAbsolutePath());
+
+ createLegacyDatabaseWithOldBot(legacyCheckout, workDir);
+ assertLegacyWalWasExercised(workDir);
+
+ RecordingFailureHandler failureHandler = new RecordingFailureHandler();
+
+ System.out.println("Current test JVM Java version: " +
System.getProperty("java.version"));
+ System.out.println("Current test JVM java.home: " +
System.getProperty("java.home"));
+
+ IgniteConfiguration currentCfg = currentNodeConfiguration(workDir,
64212, failureHandler);
+
+ assertCurrentWalConfigurationCompatible(currentCfg);
+
+ Ignite ignite = Ignition.start(currentCfg);
+
+ try {
+ System.out.println("Current Ignite node version: " +
ignite.version());
+
+ ignite.cluster().active(true);
+
+ String migrationRes = runMigrationsWithFilteredOutput(ignite);
+
+ System.out.println("Migration result: " + migrationRes);
+
+ PersistentStringCompactor compactor = new
PersistentStringCompactor(ignite);
+
+ int srvId = ITeamcityIgnited.serverIdToInt(SRV_ID);
+ long key = BuildRefDao.buildIdToCacheKey(srvId, 100500);
+
+ IgniteCache<Long, BuildRefCompacted> refs =
ignite.cache(BuildRefDao.TEAMCITY_BUILD_CACHE_NAME);
+ IgniteCache<Long, FatBuildCompacted> fatBuilds =
ignite.cache(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME);
+
+ assertNotNull(refs);
+ assertNotNull(fatBuilds);
+
+ BuildRefCompacted ref = refs.get(key);
+ FatBuildCompacted fatBuild = fatBuilds.get(key);
+
+ assertNotNull(ref);
+ assertNotNull(fatBuild);
+ assertEquals(RUN_ALL_JAVA_8, ref.buildTypeId(compactor));
+ assertEquals(BRANCH, ref.branchName(compactor));
+ assertTrue(fatBuild.isFinished(compactor));
+
+ Object migrated =
ignite.cache(LEGACY_GRID_INT_LIST_CACHE).get("legacy");
+
+ assertEquals(GridIntList.asList(1, 2, 3), migrated);
+
+ failureHandler.assertNoFailure();
+ }
+ finally {
+ failureHandler.stopping();
+
+ ignite.close();
+ }
+ }
+
+ /**
+ * @param ignite Ignite.
+ * @return Migration result string.
+ */
+ private String runMigrationsWithFilteredOutput(Ignite ignite) throws
Exception {
+ PrintStream origOut = System.out;
+ PrintStream origErr = System.err;
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+
+ try (PrintStream capture = new PrintStream(buf, true,
StandardCharsets.UTF_8.name())) {
+ System.setOut(capture);
+ System.setErr(capture);
+
+ try {
+ return new DbMigrations(ignite).dataMigration();
+ }
+ finally {
+ System.setOut(origOut);
+ System.setErr(origErr);
+
+ printMigrationSummary(new String(buf.toByteArray(),
StandardCharsets.UTF_8));
+ }
+ }
+ }
+
+ /**
+ * @param log Captured migration output.
+ */
+ private void printMigrationSummary(String log) {
+ Set<String> running = new LinkedHashSet<>();
+ Set<String> completed = new LinkedHashSet<>();
+ int missingCaches = 0;
+ Set<String> gridIntListLines = new LinkedHashSet<>();
+
+ for (String line : log.split("\\R")) {
+ if (line.startsWith("Running migration procedure ["))
+ running.add(migrationName(line));
+
+ if (line.startsWith("Completed migration procedure ["))
+ completed.add(migrationName(line));
+
+ if (line.startsWith("cache [") && line.endsWith("] not found"))
+ missingCaches++;
+
+ if (line.contains("migrate-GridIntList"))
+ gridIntListLines.add(line);
+ }
+
+ for (String line : gridIntListLines)
+ System.out.println("Migration detail: " + line);
+
+ System.out.println("Migration procedures observed: running=" +
running.size() + ", completed="
+ + completed.size() + ", missingCaches=" + missingCaches);
+ assertTrue("GridIntList migration must run",
!gridIntListLines.isEmpty());
+ }
+
+ /**
+ * @param line Migration log line.
+ * @return Migration procedure name.
+ */
+ private String migrationName(String line) {
+ int start = line.indexOf('[');
+ int end = line.lastIndexOf(']');
+
+ return start >= 0 && end > start ? line.substring(start + 1, end) :
line;
+ }
+
+ /**
+ * @param workDir Ignite work root populated by old code.
+ */
+ private void assertLegacyWalWasExercised(File workDir) throws IOException {
+ WalLayout wal = walLayout(workDir);
+
+ System.out.println("Legacy WAL layout before current start: active=" +
wal.activeFiles
+ + ", archive=" + wal.archiveFiles + ", archiveBytes=" +
wal.archiveBytes);
+
+ assertEquals("Old Ignite should create the legacy default active WAL
ring",
+ LEGACY_WAL_SEGMENTS,
+ wal.activeFiles);
+ assertTrue("Legacy generator must write enough data to roll WAL into
archive", wal.archiveFiles > 0);
+ }
+
+ /**
+ * @param cfg Current Ignite configuration.
+ */
+ private void assertCurrentWalConfigurationCompatible(IgniteConfiguration
cfg) {
+ org.apache.ignite.configuration.DataStorageConfiguration dsCfg =
cfg.getDataStorageConfiguration();
+
+ assertEquals("Current WAL segments must remain compatible with the
legacy Ignite 2.14 PDS",
+ LEGACY_WAL_SEGMENTS,
+ dsCfg.getWalSegments());
+ assertEquals("Current WAL segment size must remain compatible with the
legacy Ignite 2.14 PDS",
+ LEGACY_WAL_SEGMENT_SIZE,
+ dsCfg.getWalSegmentSize());
+ assertFalse("Old PDS does not contain checkpoint recovery files by
default",
+ dsCfg.isWriteRecoveryDataOnCheckpoint());
+ }
+
+ /**
+ * @param workDir Ignite work root.
+ * @return WAL layout.
+ */
+ private WalLayout walLayout(File workDir) throws IOException {
+ File walRoot = new File(workDir, "tcbot_srv/db/wal");
+
+ return new WalLayout(
+ countWalFiles(new File(walRoot, "tcbot")),
+ countWalFiles(new File(walRoot, "archive/tcbot")),
+ walBytes(new File(walRoot, "archive/tcbot")));
+ }
+
+ /**
+ * @param dir WAL directory.
+ * @return Number of segment files.
+ */
+ private int countWalFiles(File dir) throws IOException {
+ if (!dir.isDirectory())
+ return 0;
+
+ try (java.util.stream.Stream<Path> paths = Files.list(dir.toPath())) {
+ return (int)paths
+ .filter(Files::isRegularFile)
+ .map(path -> path.getFileName().toString())
+ .filter(name -> name.endsWith(".wal") ||
name.endsWith(".wal.zip"))
+ .count();
+ }
+ }
+
+ /**
+ * @param dir WAL directory.
+ * @return Total file size.
+ */
+ private long walBytes(File dir) throws IOException {
+ if (!dir.isDirectory())
+ return 0;
+
+ try (java.util.stream.Stream<Path> paths = Files.list(dir.toPath())) {
+ return paths
+ .filter(Files::isRegularFile)
+ .filter(path -> {
+ String name = path.getFileName().toString();
+
+ return name.endsWith(".wal") || name.endsWith(".wal.zip");
+ })
+ .mapToLong(path -> path.toFile().length())
+ .sum();
+ }
+ }
+
+ /**
+ * @param checkoutDir Directory for old source checkout.
+ * @param workDir Ignite work root to be populated by old code.
+ */
+ private void createLegacyDatabaseWithOldBot(File checkoutDir, File
workDir) throws Exception {
+ File archive = new File(tmp.getRoot(), "legacy.zip");
+
+ run(repositoryRoot(), "", "git", "archive", "--format=zip",
"--output=" + archive.getAbsolutePath(),
+ LEGACY_COMMIT);
+
+ unzip(archive, checkoutDir);
+
+ String oldIgniteVersion = legacyIgniteVersion(checkoutDir.toPath());
+
+ System.out.println("Legacy source commit-ish: " + LEGACY_COMMIT);
+ System.out.println("Legacy checkout Ignite version from build.gradle:
" + oldIgniteVersion);
+
+ assertEquals(LEGACY_IGNITE_VERSION, oldIgniteVersion);
+
+ normalizeLegacyGradleBuild(checkoutDir.toPath());
+ addLegacyGeneratorModule(checkoutDir.toPath());
+
+ String gradle = legacyGradleCommand();
+ String javaHome = legacyJavaHome();
+
+ System.out.println("Legacy Gradle command: " + gradle);
+ System.out.println("Legacy Java home: " + javaHome);
+
+ run(javaHomeBin(javaHome, "java").getParentFile(),
"[legacy-java-version] ",
+ javaHomeBin(javaHome, "java").getAbsolutePath(), "-version");
+
+ int major = javaMajor(javaHome);
+
+ if (major != 8 && major != 11)
+ throw new AssertionError("Legacy database must be generated with
Java 8 or 11, but got Java " + major
+ + " from " + javaHome);
+
+ List<String> cmd = new ArrayList<>();
+
+ if (isWindows() && gradle.endsWith(".bat")) {
+ cmd.add("cmd");
+ cmd.add("/c");
+ }
+
+ cmd.add(gradle);
+ cmd.add("--no-daemon");
+ cmd.add("-Dorg.gradle.java.home=" + javaHome);
+ cmd.add(":legacy-db-generator:run");
+ cmd.add("--args=" + workDir.getAbsolutePath());
+
+ ProcessBuilder pb = new ProcessBuilder(cmd);
+ pb.directory(checkoutDir);
+ pb.redirectErrorStream(true);
+ pb.environment().put("JAVA_HOME", javaHome);
+ pb.environment().put("PATH", new File(javaHome,
"bin").getAbsolutePath() + File.pathSeparator
+ + pb.environment().get("PATH"));
+
+ run(pb, "[legacy-gradle] ");
+ }
+
+ /**
+ * @param root Legacy checkout root.
+ */
+ private void normalizeLegacyGradleBuild(Path root) throws IOException {
+ Files.walk(root)
+ .filter(path -> path.getFileName().toString().endsWith(".gradle"))
+ .forEach(path -> {
+ try {
+ String text = new String(Files.readAllBytes(path),
StandardCharsets.UTF_8);
+
+ text = text.replace("apply plugin: 'java'", "apply plugin:
'java-library'")
+ .replace("testCompile", "testImplementation")
+ .replaceAll("(?m)^(\\s*)compile(\\s+|\\()", "$1api$2");
+
+ Files.write(path, text.getBytes(StandardCharsets.UTF_8));
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ /**
+ * @param root Legacy checkout root.
+ */
+ private void addLegacyGeneratorModule(Path root) throws IOException {
+ Files.write(root.resolve("settings.gradle"),
+ "\ninclude
'legacy-db-generator'\n".getBytes(StandardCharsets.UTF_8),
+ java.nio.file.StandardOpenOption.APPEND);
+
+ Path module = root.resolve("legacy-db-generator");
+ Path javaDir = module.resolve("src/main/java/org/apache/ignite/ci/db");
+ Path resourcesDir = module.resolve("src/main/resources");
+
+ Files.createDirectories(javaDir);
+ Files.createDirectories(resourcesDir);
+
+ Files.write(module.resolve("build.gradle"),
legacyGeneratorBuildGradle().getBytes(StandardCharsets.UTF_8));
+ Files.write(javaDir.resolve("LegacyBotDatabaseGenerator.java"),
+ legacyGeneratorJava().getBytes(StandardCharsets.UTF_8));
+ Files.write(resourcesDir.resolve("logback.xml"),
legacyGeneratorLogback().getBytes(StandardCharsets.UTF_8));
+ }
+
+ /**
+ * @return Build file for the injected old-source generator.
+ */
+ private String legacyGeneratorBuildGradle() {
+ return "apply plugin: 'java'\n"
+ + "apply plugin: 'application'\n"
+ + "\n"
+ + "dependencies {\n"
+ + " implementation project(':ignite-tc-helper-web')\n"
+ + " implementation project(':tcbot-common')\n"
+ + " implementation project(':tcbot-persistence')\n"
+ + " implementation project(':tcbot-teamcity')\n"
+ + " implementation project(':tcbot-teamcity-ignited')\n"
+ + " implementation group: 'com.google.guava', name: 'guava',
version: guavaVer\n"
+ + " implementation group: 'org.apache.ignite', name:
'ignite-core', version: ignVer\n"
+ + " implementation group: 'org.apache.ignite', name:
'ignite-slf4j', version: ignVer\n"
+ + " implementation group: 'org.apache.ignite', name:
'ignite-direct-io', version: ignVer\n"
+ + " implementation group: 'ch.qos.logback', name:
'logback-core', version: logbackVer\n"
+ + " implementation group: 'ch.qos.logback', name:
'logback-classic', version: logbackVer\n"
+ + "}\n"
+ + "\n"
+ + "application {\n"
+ + " mainClass =
'org.apache.ignite.ci.db.LegacyBotDatabaseGenerator'\n"
+ + " applicationDefaultJvmArgs = [\n"
+ + " '-XX:+IgnoreUnrecognizedVMOptions',\n"
+ + "
'--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED',\n"
+ + " '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED',\n"
+ + "
'--add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED',\n"
+ + "
'--add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED',\n"
+ + " '--illegal-access=permit'\n"
+ + " ]\n"
+ + "}\n";
+ }
+
+ /**
+ * @return Java source compiled and executed in the old checkout.
+ */
+ private String legacyGeneratorJava() {
+ return "package org.apache.ignite.ci.db;\n"
+ + "\n"
+ + "import java.io.File;\n"
+ + "import java.util.Collections;\n"
+ + "import java.util.Iterator;\n"
+ + "import javax.cache.Cache;\n"
+ + "import org.apache.ignite.Ignite;\n"
+ + "import org.apache.ignite.IgniteCache;\n"
+ + "import org.apache.ignite.Ignition;\n"
+ + "import org.apache.ignite.cache.QueryEntity;\n"
+ + "import org.apache.ignite.cache.query.ScanQuery;\n"
+ + "import
org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;\n"
+ + "import
org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;\n"
+ + "import org.apache.ignite.configuration.CacheConfiguration;\n"
+ + "import
org.apache.ignite.configuration.DataRegionConfiguration;\n"
+ + "import org.apache.ignite.configuration.IgniteConfiguration;\n"
+ + "import org.apache.ignite.internal.util.GridIntList;\n"
+ + "import org.apache.ignite.logger.slf4j.Slf4jLogger;\n"
+ + "import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;\n"
+ + "import org.apache.ignite.tcbot.persistence.CacheConfigs;\n"
+ + "import org.apache.ignite.tcbot.persistence.IStringCompactor;\n"
+ + "import org.apache.ignite.tcignited.ITeamcityIgnited;\n"
+ + "import org.apache.ignite.tcignited.build.FatBuildDao;\n"
+ + "import org.apache.ignite.tcignited.buildref.BuildRefDao;\n"
+ + "import org.apache.ignite.tcservice.model.conf.BuildType;\n"
+ + "import org.apache.ignite.tcservice.model.hist.BuildRef;\n"
+ + "import org.apache.ignite.tcservice.model.result.Build;\n"
+ + "\n"
+ + "public class LegacyBotDatabaseGenerator {\n"
+ + " private static final String SRV_ID = \"" + SRV_ID + "\";\n"
+ + " private static final String RUN_ALL_JAVA_8 = \"" +
RUN_ALL_JAVA_8 + "\";\n"
+ + " private static final String BRANCH = \"" + BRANCH + "\";\n"
+ + " private static final String LEGACY_GRID_INT_LIST_CACHE =
\"" + LEGACY_GRID_INT_LIST_CACHE + "\";\n"
+ + " private static final String LEGACY_WAL_STRESS_CACHE = \"" +
LEGACY_WAL_STRESS_CACHE + "\";\n"
+ + " private static final long REGION_SIZE = " + REGION_SIZE +
"L;\n"
+ + " private static final int WAL_STRESS_MB = " +
legacyWalStressMb() + ";\n"
+ + "\n"
+ + " public static void main(String[] args) throws Exception {\n"
+ + " if (args.length != 1)\n"
+ + " throw new IllegalArgumentException(\"Expected work
dir argument\");\n"
+ + "\n"
+ + " System.out.println(\"Legacy generator Java version: \"
+ System.getProperty(\"java.version\"));\n"
+ + " System.out.println(\"Legacy generator java.home: \" +
System.getProperty(\"java.home\"));\n"
+ + "\n"
+ + " File workDir = new File(args[0]);\n"
+ + "\n"
+ + " try (Ignite ignite =
Ignition.start(configuration(workDir))) {\n"
+ + " System.out.println(\"Legacy Ignite node version: \"
+ ignite.version());\n"
+ + "\n"
+ + " ignite.cluster().active(true);\n"
+ + "\n"
+ + " PersistentStringCompactor compactor = new
PersistentStringCompactor(ignite);\n"
+ + " IgniteCache<Long, BuildRefCompacted> refs =
ignite.getOrCreateCache(CacheConfigs.getCacheV2Config(BuildRefDao.TEAMCITY_BUILD_CACHE_NAME));\n"
+ + " IgniteCache<Long, FatBuildCompacted> fatBuilds =
ignite.getOrCreateCache(CacheConfigs.getCacheV2Config(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME));\n"
+ + " int srvId =
ITeamcityIgnited.serverIdToInt(SRV_ID);\n"
+ + "\n"
+ + " for (int i = 0; i < 3; i++) {\n"
+ + " Build build = fakeJava8Build(100500 + i);\n"
+ + " long key = BuildRefDao.buildIdToCacheKey(srvId,
build.getId());\n"
+ + " refs.put(key, new BuildRefCompacted(compactor,
build));\n"
+ + " fatBuilds.put(key, new
FatBuildCompacted(compactor, build));\n"
+ + " }\n"
+ + "\n"
+ + " IgniteCache<String, Object> legacy =
ignite.getOrCreateCache(new CacheConfiguration<String,
Object>(LEGACY_GRID_INT_LIST_CACHE));\n"
+ + " legacy.put(\"legacy\", new GridIntList(new int[]
{1, 2, 3}));\n"
+ + "\n"
+ + " writeWalStressData(ignite);\n"
+ + "\n"
+ + " System.out.println(\"Legacy DB generated at: \" +
workDir.getAbsolutePath());\n"
+ + " }\n"
+ + " }\n"
+ + "\n"
+ + " private static void writeWalStressData(Ignite ignite) {\n"
+ + " IgniteCache<Integer, byte[]> cache =
ignite.getOrCreateCache(new CacheConfiguration<Integer,
byte[]>(LEGACY_WAL_STRESS_CACHE));\n"
+ + " byte[] payload = new byte[1024 * 1024];\n"
+ + "\n"
+ + " for (int i = 0; i < payload.length; i++)\n"
+ + " payload[i] = (byte)i;\n"
+ + "\n"
+ + " for (int i = 0; i < WAL_STRESS_MB; i++) {\n"
+ + " payload[0] = (byte)i;\n"
+ + " cache.put(i % 32, payload.clone());\n"
+ + "\n"
+ + " if ((i + 1) % 128 == 0)\n"
+ + " System.out.println(\"Legacy WAL stress wrote \"
+ (i + 1) + \" MB\");\n"
+ + " }\n"
+ + "\n"
+ + " System.out.println(\"Legacy WAL stress total payload:
\" + WAL_STRESS_MB + \" MB\");\n"
+ + " }\n"
+ + "\n"
+ + " private static IgniteConfiguration configuration(File
workDir) throws Exception {\n"
+ + " IgniteConfiguration cfg = new IgniteConfiguration();\n"
+ + " Ignite2Configurer.setIgniteHome(cfg, workDir);\n"
+ + " cfg.setWorkDirectory(new File(cfg.getIgniteHome(),
\"tcbot_srv\").getCanonicalPath());\n"
+ + " cfg.setIgniteInstanceName(\"legacy-tcbot-db\");\n"
+ + " cfg.setConsistentId(\"tcbot\");\n"
+ + " cfg.setGridLogger(new Slf4jLogger());\n"
+ + "\n"
+ + " TcpDiscoverySpi spi = new TcpDiscoverySpi();\n"
+ + " spi.setLocalPort(64211);\n"
+ + " spi.setLocalPortRange(1);\n"
+ + " spi.setIpFinder(new
TcHelperDb.LocalOnlyTcpDiscoveryIpFinder(64211));\n"
+ + " cfg.setDiscoverySpi(spi);\n"
+ + "\n"
+ + " DataRegionConfiguration regConf =
Ignite2Configurer.getDataRegionConfiguration();\n"
+ + " regConf.setMaxSize(REGION_SIZE);\n"
+ + "
cfg.setDataStorageConfiguration(Ignite2Configurer.getDataStorageConfiguration(regConf));\n"
+ + "\n"
+ + " return cfg;\n"
+ + " }\n"
+ + "\n"
+ + " private static Build fakeJava8Build(int buildId) {\n"
+ + " Build build = new Build();\n"
+ + " build.setId(buildId);\n"
+ + " build.buildTypeId = RUN_ALL_JAVA_8;\n"
+ + " build.branchName = BRANCH;\n"
+ + " build.status = BuildRef.STATUS_SUCCESS;\n"
+ + " build.state = BuildRef.STATE_FINISHED;\n"
+ + " build.defaultBranch = Boolean.FALSE;\n"
+ + " build.composite = Boolean.TRUE;\n"
+ + " build.webUrl =
\"http://localhost/perf-test/teamcity/build/\"; + buildId;\n"
+ + " build.setQueuedDateTs(1700000000000L + buildId);\n"
+ + " build.setStartDateTs(1700000010000L + buildId);\n"
+ + " build.setFinishDateTs(1700000070000L + buildId);\n"
+ + "\n"
+ + " BuildType buildType = new BuildType();\n"
+ + " buildType.setId(RUN_ALL_JAVA_8);\n"
+ + " buildType.setName(\"Ignite Java 8 Run All\");\n"
+ + " buildType.setProjectId(\"IgniteTests24Java8\");\n"
+ + " buildType.setProjectName(\"Ignite Tests Java 8\");\n"
+ + "
buildType.setWebUrl(\"http://localhost/perf-test/teamcity/buildConfiguration/\";
+ RUN_ALL_JAVA_8);\n"
+ + " build.setBuildType(buildType);\n"
+ + "\n"
+ + " return build;\n"
+ + " }\n"
+ + "\n"
+ + " private static class PersistentStringCompactor implements
IStringCompactor {\n"
+ + " private final IgniteCache<String,
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity>
stringsCache;\n"
+ + " private int nextId;\n"
+ + "\n"
+ + " private PersistentStringCompactor(Ignite ignite) {\n"
+ + " CacheConfiguration<String,
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity>
cfg =
CacheConfigs.getCache8PartsConfig(org.apache.ignite.tcbot.persistence.IgniteStringCompactor.STRINGS_CACHE);\n"
+ + " cfg.setQueryEntities(Collections.singletonList(new
QueryEntity(String.class,
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity.class)));\n"
+ + " stringsCache = ignite.getOrCreateCache(cfg);\n"
+ + " nextId = maxKnownId() + 1;\n"
+ + " }\n"
+ + "\n"
+ + " @Override public int getStringId(String val) {\n"
+ + " if (val == null)\n"
+ + " return STRING_NULL;\n"
+ + "
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity
entity = stringsCache.get(val);\n"
+ + " if (entity != null)\n"
+ + " return entity.id();\n"
+ + " int id = nextId++;\n"
+ + " stringsCache.put(val, new
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity(id,
val));\n"
+ + " return id;\n"
+ + " }\n"
+ + "\n"
+ + " @Override public String getStringFromId(int id) {\n"
+ + " if (id < 0)\n"
+ + " return null;\n"
+ + " Iterator<Cache.Entry<String,
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity>>
it = stringsCache.query(new ScanQuery<String,
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity>((key,
val) -> val.id() == id)).iterator();\n"
+ + " return it.hasNext() ? it.next().getValue().val() :
null;\n"
+ + " }\n"
+ + "\n"
+ + " @Override public Integer getStringIdIfPresent(String
val) {\n"
+ + " if (val == null)\n"
+ + " return STRING_NULL;\n"
+ + "
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity
entity = stringsCache.get(val);\n"
+ + " return entity == null ? null : entity.id();\n"
+ + " }\n"
+ + "\n"
+ + " private int maxKnownId() {\n"
+ + " int max = 0;\n"
+ + " for (Cache.Entry<String,
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity> e
: stringsCache)\n"
+ + " max = Math.max(max, e.getValue().id());\n"
+ + " return max;\n"
+ + " }\n"
+ + " }\n"
+ + "}\n";
+ }
+
+ /**
+ * @return Quiet Logback configuration for the injected old-source
generator.
+ */
+ private String legacyGeneratorLogback() {
+ return "<configuration>\n"
+ + " <appender name=\"STDOUT\"
class=\"ch.qos.logback.core.ConsoleAppender\">\n"
+ + " <encoder><pattern>%msg%n</pattern></encoder>\n"
+ + " </appender>\n"
+ + " <root level=\"WARN\">\n"
+ + " <appender-ref ref=\"STDOUT\" />\n"
+ + " </root>\n"
+ + "</configuration>\n";
+ }
+
+ /**
+ * @return How much old Ignite should write to WAL, in megabytes.
+ */
+ private int legacyWalStressMb() {
+ return Integer.getInteger("compat.legacy.wal.stress.mb", 704);
+ }
+
+ /**
+ * @param workDir Test work dir.
+ * @param port Discovery port.
+ * @param failureHandler Failure handler.
+ */
+ private IgniteConfiguration currentNodeConfiguration(
+ File workDir,
+ int port,
+ RecordingFailureHandler failureHandler) throws IOException {
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ Ignite2Configurer.setIgniteHome(cfg, workDir);
+ cfg.setWorkDirectory(new File(cfg.getIgniteHome(),
"tcbot_srv").getCanonicalPath());
+ cfg.setIgniteInstanceName("current-tcbot-db");
+ cfg.setConsistentId("tcbot");
+ cfg.setGridLogger(new Slf4jLogger());
+ cfg.setFailureHandler(failureHandler);
+
+ TcpDiscoverySpi spi = new TcpDiscoverySpi();
+ spi.setLocalPort(port);
+ spi.setLocalPortRange(1);
+ spi.setIpFinder(new TcHelperDb.LocalOnlyTcpDiscoveryIpFinder(port));
+ cfg.setDiscoverySpi(spi);
+
+ DataRegionConfiguration regConf =
Ignite2Configurer.getDataRegionConfiguration();
+ regConf.setMaxSize(REGION_SIZE);
+
+
cfg.setDataStorageConfiguration(Ignite2Configurer.getDataStorageConfiguration(regConf));
+
+ return cfg;
+ }
+
+ /**
+ * @param archive Zip archive.
+ * @param target Target directory.
+ */
+ private void unzip(File archive, File target) throws IOException {
+ try (ZipInputStream zip = new ZipInputStream(new
FileInputStream(archive))) {
+ ZipEntry entry;
+
+ while ((entry = zip.getNextEntry()) != null) {
+ File out = new File(target, entry.getName());
+ String targetPath = target.getCanonicalPath();
+ String outPath = out.getCanonicalPath();
+
+ if (!outPath.startsWith(targetPath + File.separator) &&
!Objects.equals(outPath, targetPath))
+ throw new IOException("Zip entry escapes target dir: " +
entry.getName());
+
+ if (entry.isDirectory()) {
+ Files.createDirectories(out.toPath());
+
+ continue;
+ }
+
+ Files.createDirectories(out.getParentFile().toPath());
+
+ try (FileOutputStream fos = new FileOutputStream(out)) {
+ byte[] buf = new byte[64 * 1024];
+ int read;
+
+ while ((read = zip.read(buf)) > 0)
+ fos.write(buf, 0, read);
+ }
+ }
+ }
+ }
+
+ /**
+ * @return Gradle executable for building the old checkout.
+ */
+ private String legacyGradleCommand() {
+ String prop = System.getProperty("legacy.gradle.cmd");
+
+ if (prop != null && !prop.isEmpty())
+ return prop;
+
+ String env = System.getenv("LEGACY_GRADLE_CMD");
+
+ if (env != null && !env.isEmpty())
+ return env;
+
+ File bundled = new File("C:/dev_env/gradle-7.6.6/bin/" + (isWindows()
? "gradle.bat" : "gradle"));
+
+ if (!bundled.isFile())
+ throw new IllegalStateException("Legacy Gradle is required. Set
-Dlegacy.gradle.cmd or LEGACY_GRADLE_CMD.");
+
+ return bundled.getAbsolutePath();
+ }
+
+ /**
+ * @return Java home for building/running old bot code.
+ */
+ private String legacyJavaHome() {
+ String prop = System.getProperty("legacy.java.home");
+
+ if (prop != null && !prop.isEmpty())
+ return prop;
+
+ String env = System.getenv("LEGACY_JAVA_HOME");
+
+ if (env != null && !env.isEmpty())
+ return env;
+
+ File bundled = new File("C:/dev_env/jdk-11.0.31.11-hotspot");
+
+ if (!bundled.isDirectory())
+ throw new IllegalStateException("Legacy Java 8/11 is required. Set
-Dlegacy.java.home or LEGACY_JAVA_HOME.");
+
+ return bundled.getAbsolutePath();
+ }
+
+ /**
+ * @param javaHome Java home.
+ * @return Major Java version.
+ */
+ private int javaMajor(String javaHome) throws Exception {
+ ProcessBuilder pb = new ProcessBuilder(javaHomeBin(javaHome,
"java").getAbsolutePath(), "-XshowSettings:properties",
+ "-version");
+ pb.redirectErrorStream(true);
+
+ Process proc = pb.start();
+ String version = null;
+
+ try (BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream(),
+ StandardCharsets.UTF_8))) {
+ String line;
+
+ while ((line = reader.readLine()) != null) {
+ int idx = line.indexOf("java.version = ");
+
+ if (idx >= 0)
+ version = line.substring(idx + "java.version =
".length()).trim();
+ }
+ }
+
+ int code = proc.waitFor();
+
+ if (code != 0)
+ throw new AssertionError("Unable to detect legacy Java version
from " + javaHome);
+
+ if (version == null)
+ throw new AssertionError("java.version was not printed by " +
javaHome);
+
+ if (version.startsWith("1."))
+ return Integer.parseInt(version.substring(2, 3));
+
+ int dot = version.indexOf('.');
+
+ return Integer.parseInt(dot > 0 ? version.substring(0, dot) : version);
+ }
+
+ /**
+ * @param javaHome Java home.
+ * @param executable Executable name without extension.
+ * @return Java executable.
+ */
+ private File javaHomeBin(String javaHome, String executable) {
+ return new File(new File(javaHome, "bin"), executable + (isWindows() ?
".exe" : ""));
+ }
+
+ /**
+ * @param dir Work dir.
+ * @param prefix Prefix for process output.
+ * @param cmd Command.
+ */
+ private void run(File dir, String prefix, String... cmd) throws Exception {
+ ProcessBuilder pb = new ProcessBuilder(cmd);
+ pb.directory(dir);
+ pb.redirectErrorStream(true);
+
+ run(pb, prefix);
+ }
+
+ /**
+ * @param pb Process builder.
+ * @param prefix Prefix for process output.
+ */
+ private void run(ProcessBuilder pb, String prefix) throws Exception {
+ Process proc = pb.start();
+ List<String> tail = new ArrayList<>();
+
+ try (BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream(),
+ StandardCharsets.UTF_8))) {
+ String line;
+
+ while ((line = reader.readLine()) != null) {
+ if (shouldPrintProcessLine(prefix, line))
+ System.out.println(prefix + line);
+
+ tail.add(line);
+
+ if (tail.size() > 120)
+ tail.remove(0);
+ }
+ }
+
+ int code = proc.waitFor();
+
+ if (code != 0)
+ throw new AssertionError("Command failed with code " + code + ": "
+ pb.command() + "\n"
+ + String.join("\n", tail));
+ }
+
+ /**
+ * @param prefix Process output prefix.
+ * @param line Process output line.
+ * @return {@code true} if the line is useful enough for normal successful
test output.
+ */
+ private boolean shouldPrintProcessLine(String prefix, String line) {
+ if (!"[legacy-gradle] ".equals(prefix))
+ return true;
+
+ return line.startsWith("> Task")
+ || line.startsWith("BUILD ")
+ || line.contains("Legacy generator")
+ || line.contains("Legacy WAL stress")
+ || line.contains("Legacy Ignite node version")
+ || line.contains("Legacy DB generated")
+ || line.contains("ver. " + LEGACY_IGNITE_VERSION)
+ || line.contains("FAILED")
+ || line.contains("ERROR")
+ || line.contains("Exception");
+ }
+
+ /**
+ * @param root Legacy checkout root.
+ * @return Ignite version declared by the old checkout.
+ */
+ private String legacyIgniteVersion(Path root) throws IOException {
+ String build = new
String(Files.readAllBytes(root.resolve("build.gradle")),
StandardCharsets.UTF_8);
+
+ for (String line : build.split("\\R")) {
+ String trimmed = line.trim();
+
+ if (trimmed.startsWith("ignVer")) {
+ int start = trimmed.indexOf('\'');
+ int end = trimmed.lastIndexOf('\'');
+
+ if (start >= 0 && end > start)
+ return trimmed.substring(start + 1, end);
+ }
+ }
+
+ throw new IOException("Unable to find ignVer in " +
root.resolve("build.gradle"));
+ }
+
+ /**
+ * @return Stable ignored work root where generated PDS is left for
inspection.
+ */
+ private File compatibilityWorkRoot() {
+ String prop = System.getProperty("compat.work.dir");
+
+ if (prop != null && !prop.isEmpty())
+ return new File(prop).getAbsoluteFile();
+
+ return new File(repositoryRoot(),
"migrator/src/test/work/ignite-db-compat").getAbsoluteFile();
+ }
+
+ /**
+ * @param allowedRoot Root that may contain deleted files.
+ * @param dir Directory to recreate.
+ */
+ private void recreateDirectory(File allowedRoot, File dir) throws
IOException {
+ String rootPath = allowedRoot.getCanonicalPath();
+ String dirPath = dir.getCanonicalPath();
+
+ if (!dirPath.startsWith(rootPath + File.separator))
+ throw new IOException("Refusing to recreate directory outside
compat work root: " + dirPath);
+
+ if (dir.exists()) {
+ Files.walk(dir.toPath())
+ .sorted(Comparator.reverseOrder())
+ .forEach(path -> {
+ try {
+ Files.delete(path);
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ Files.createDirectories(dir.toPath());
+ }
+
+ /**
+ * @return Repository root. Gradle may run this test from a subproject
directory.
+ */
+ private File repositoryRoot() {
+ File cur = new File(System.getProperty("user.dir")).getAbsoluteFile();
+
+ while (cur != null) {
+ if (new File(cur, ".git").exists())
+ return cur;
+
+ cur = cur.getParentFile();
+ }
+
+ throw new IllegalStateException("Unable to locate git repository root
from " + System.getProperty("user.dir"));
+ }
+
+ /**
+ * @return {@code true} on Windows.
+ */
+ private boolean isWindows() {
+ return System.getProperty("os.name").toLowerCase().contains("win");
+ }
+
+ /**
+ * Records startup/runtime failures without letting Ignite halt the Gradle
worker during test shutdown.
+ */
+ private static class RecordingFailureHandler implements FailureHandler {
+ /** */
+ private final List<FailureContext> failures =
Collections.synchronizedList(new ArrayList<>());
+
+ /** */
+ private volatile boolean stopping;
+
+ /** {@inheritDoc} */
+ @Override public boolean onFailure(Ignite ignite, FailureContext
failureCtx) {
+ if (!stopping)
+ failures.add(failureCtx);
+
+ return false;
+ }
+
+ /** */
+ private void stopping() {
+ stopping = true;
+ }
+
+ /** */
+ private void assertNoFailure() {
+ if (!failures.isEmpty())
+ throw new AssertionError("Ignite failure before shutdown: " +
failures);
+ }
+ }
+
+ /**
+ * WAL files left by a generated persistent store.
+ */
+ private static class WalLayout {
+ /** */
+ private final int activeFiles;
+
+ /** */
+ private final int archiveFiles;
+
+ /** */
+ private final long archiveBytes;
+
+ /**
+ * @param activeFiles Active WAL segment files.
+ * @param archiveFiles Archived WAL segment files.
+ * @param archiveBytes Archived WAL bytes.
+ */
+ private WalLayout(int activeFiles, int archiveFiles, long
archiveBytes) {
+ this.activeFiles = activeFiles;
+ this.archiveFiles = archiveFiles;
+ this.archiveBytes = archiveBytes;
+ }
+ }
+
+ /**
+ * Minimal persistent compactor for reading bot-shaped cache data without
starting external services.
+ */
+ private static class PersistentStringCompactor implements IStringCompactor
{
+ /** */
+ private final IgniteCache<String,
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity>
+ stringsCache;
+
+ /** */
+ private int nextId;
+
+ /**
+ * @param ignite Ignite.
+ */
+ private PersistentStringCompactor(Ignite ignite) {
+ org.apache.ignite.configuration.CacheConfiguration<String,
+
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity>
cfg =
+ CacheConfigs.getCache8PartsConfig(STRINGS_CACHE);
+
+ cfg.setQueryEntities(Collections.singletonList(new QueryEntity(
+ String.class,
+
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity.class)));
+
+ stringsCache = ignite.getOrCreateCache(cfg);
+
+ nextId = maxKnownId() + 1;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getStringId(String val) {
+ if (val == null)
+ return STRING_NULL;
+
+
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity
entity = stringsCache.get(val);
+
+ if (entity != null)
+ return entity.id();
+
+ int id = nextId++;
+
+ stringsCache.put(val, new
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity(
+ id, val));
+
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getStringFromId(int id) {
+ if (id < 0)
+ return null;
+
+ Iterator<Cache.Entry<String,
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity>>
+ it = stringsCache.query(new ScanQuery<String,
+
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity>(
+ (key, val) -> val.id() == id)).iterator();
+
+ return it.hasNext() ? it.next().getValue().val() : null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Integer getStringIdIfPresent(String val) {
+ if (val == null)
+ return STRING_NULL;
+
+
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity
entity = stringsCache.get(val);
+
+ return entity == null ? null : entity.id();
+ }
+
+ /** */
+ private int maxKnownId() {
+ int max = 0;
+
+ for (Cache.Entry<String,
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity> e :
+ stringsCache)
+ max = Math.max(max, e.getValue().id());
+
+ return max;
+ }
+ }
+}
diff --git a/migrator/src/test/resources/logback-test.xml
b/migrator/src/test/resources/logback-test.xml
new file mode 100644
index 00000000..309f4c04
--- /dev/null
+++ b/migrator/src/test/resources/logback-test.xml
@@ -0,0 +1,11 @@
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="WARN">
+ <appender-ref ref="STDOUT" />
+ </root>
+</configuration>
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
index bf55a4a8..8e04a280 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -170,6 +171,7 @@ public class BuildChainProcessor {
Map<Integer, Future<FatBuildCompacted>> builds =
loadAllBuildsInChains(entryPoints, mode, tcIgn);
Map<String, List<Future<FatBuildCompacted>>> freshRebuilds = new
ConcurrentHashMap<>();
+ Map<String, Map<String, ScheduledBuildsCount>> scheduledCountsByBranch
= new ConcurrentHashMap<>();
groupByBuildType(builds).forEach(
(k, buildsForBt) -> {
@@ -208,7 +210,7 @@ public class BuildChainProcessor {
analyzeTests(ctx, tcIgn, procLog);
- fillBuildCounts(ctx, tcIgn, includeScheduledInfo);
+ fillBuildCounts(ctx, tcIgn, includeScheduledInfo,
scheduledCountsByBranch);
contexts.add(ctx);
});
@@ -425,25 +427,66 @@ public class BuildChainProcessor {
@AutoProfiling
protected void fillBuildCounts(MultBuildRunCtx outCtx,
ITeamcityIgnited teamcityIgnited, boolean includeScheduledInfo) {
- if (includeScheduledInfo && !outCtx.hasScheduledBuildsInfo()) {
- List<BuildRefCompacted> runAllBuilds =
teamcityIgnited.getAllBuildsCompacted(outCtx.suiteId(), outCtx.branchName());
+ fillBuildCounts(outCtx, teamcityIgnited, includeScheduledInfo, new
ConcurrentHashMap<>());
+ }
- long cntRunning = runAllBuilds
- .stream()
- .filter(r -> r.isNotCancelled(compactor))
- .filter(r -> r.isRunning(compactor)).count();
+ @SuppressWarnings("WeakerAccess")
+ @AutoProfiling
+ protected void fillBuildCounts(MultBuildRunCtx outCtx,
+ ITeamcityIgnited teamcityIgnited,
+ boolean includeScheduledInfo,
+ Map<String, Map<String, ScheduledBuildsCount>>
scheduledCountsByBranch) {
+ if (includeScheduledInfo && !outCtx.hasScheduledBuildsInfo()) {
+ ScheduledBuildsCount counts = scheduledCountsByBranch
+ .computeIfAbsent(outCtx.branchName(), branch ->
scheduledBuildCountsBySuite(teamcityIgnited, branch))
+ .get(outCtx.suiteId());
- outCtx.setRunningBuildCount((int)cntRunning);
+ if (counts == null) {
+ outCtx.setRunningBuildCount(0);
+ outCtx.setQueuedBuildCount(0);
- long cntQueued = runAllBuilds
- .stream()
- .filter(r -> r.isNotCancelled(compactor))
- .filter(r -> r.isQueued(compactor)).count();
+ return;
+ }
- outCtx.setQueuedBuildCount((int)cntQueued);
+ outCtx.setRunningBuildCount(counts.running);
+ outCtx.setQueuedBuildCount(counts.queued);
}
}
+ /**
+ * @param teamcityIgnited TeamCity service.
+ * @param branch Branch.
+ */
+ private Map<String, ScheduledBuildsCount>
scheduledBuildCountsBySuite(ITeamcityIgnited teamcityIgnited,
+ String branch) {
+ Map<String, ScheduledBuildsCount> res = new HashMap<>();
+
+ teamcityIgnited.getQueuedAndRunningBuildsCompacted(branch).stream()
+ .filter(ref -> ref.isNotCancelled(compactor))
+ .forEach(ref -> {
+ String suiteId = ref.buildTypeId(compactor);
+ ScheduledBuildsCount counts = res.computeIfAbsent(suiteId, k
-> new ScheduledBuildsCount());
+
+ if (ref.isRunning(compactor))
+ counts.running++;
+ else if (ref.isQueued(compactor))
+ counts.queued++;
+ });
+
+ return res;
+ }
+
+ /**
+ * Queued and running builds count.
+ */
+ private static class ScheduledBuildsCount {
+ /** Running count. */
+ private int running;
+
+ /** Queued count. */
+ private int queued;
+ }
+
@SuppressWarnings("WeakerAccess")
@AutoProfiling
protected void analyzeTests(MultBuildRunCtx outCtx, ITeamcityIgnited
teamcity,
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/BranchTicketMatcher.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/BranchTicketMatcher.java
index fa36b777..322bc889 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/BranchTicketMatcher.java
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/BranchTicketMatcher.java
@@ -19,6 +19,7 @@ package org.apache.ignite.tcbot.engine.pr;
import com.google.common.base.Strings;
import java.util.Collection;
+import java.util.Map;
import java.util.Objects;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -91,6 +92,17 @@ public class BranchTicketMatcher {
*/
@Nullable public Ticket
resolveTicketIdForPrBasedContrib(Collection<Ticket> tickets,
IJiraServerConfig jiraCfg, String prTitle) {
+ return resolveTicketIdForPrBasedContrib(tickets, null, jiraCfg,
prTitle);
+ }
+
+ /**
+ * @param tickets Tickets.
+ * @param ticketsByKey Optional ticket lookup by full JIRA key.
+ * @param jiraCfg Jira config.
+ * @param prTitle PR title or branch name.
+ */
+ @Nullable public Ticket
resolveTicketIdForPrBasedContrib(Collection<Ticket> tickets,
+ @Nullable Map<String, Ticket> ticketsByKey, IJiraServerConfig jiraCfg,
String prTitle) {
String branchNumPrefix = jiraCfg.branchNumPrefix();
if (Strings.isNullOrEmpty(branchNumPrefix)) {
@@ -99,6 +111,12 @@ public class BranchTicketMatcher {
final String ticketKey = findFixPrefixedNumber(prTitle,
jiraPrefix);
+ if (ticketsByKey != null) {
+ Ticket ticket = ticketsByKey.get(ticketKey);
+
+ return ticket != null ? ticket : new Ticket(ticketKey);
+ }
+
return tickets.stream()
.filter(t -> Objects.equals(t.key, ticketKey))
.findFirst()
@@ -110,7 +128,7 @@ public class BranchTicketMatcher {
if (branchNum == null) // PR does not mention
return null;
- return findTicketMentions(tickets, branchNum);
+ return findTicketMentions(tickets, ticketsByKey, branchNum);
}
/**
@@ -129,9 +147,26 @@ public class BranchTicketMatcher {
* @param branchNum Branch number to be checked.
*/
@Nullable private Ticket findTicketMentions(Collection<Ticket> tickets,
@Nullable String branchNum) {
+ return findTicketMentions(tickets, null, branchNum);
+ }
+
+ /**
+ * @param tickets Tickets.
+ * @param ticketsByKey Optional ticket lookup by full JIRA key.
+ * @param branchNum Branch number to be checked.
+ */
+ @Nullable private Ticket findTicketMentions(Collection<Ticket> tickets,
+ @Nullable Map<String, Ticket> ticketsByKey, @Nullable String
branchNum) {
if (Strings.isNullOrEmpty(branchNum))
return null;
+ if (ticketsByKey != null) {
+ Ticket ticket = ticketsByKey.get(branchNum);
+
+ if (ticket != null)
+ return ticket;
+ }
+
return tickets.stream()
.filter(t -> Objects.equals(t.key, branchNum))
.findFirst()
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java
index 71ab3f43..63c89686 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
@@ -54,6 +55,8 @@ import org.apache.ignite.tcignited.SyncMode;
import org.apache.ignite.tcignited.build.UpdateCountersStorage;
import org.apache.ignite.tcignited.buildref.BranchEquivalence;
import org.apache.ignite.tcignited.creds.ICredentialsProv;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static com.google.common.base.Strings.isNullOrEmpty;
@@ -61,6 +64,13 @@ import static com.google.common.base.Strings.isNullOrEmpty;
* Process failures for some setup tracked branch, which may be
triggered/monitored by TC Bot.
*/
public class TrackedBranchChainsProcessor implements
IDetailedStatusForTrackedBranch {
+ /** Logger. */
+ private static final Logger logger =
LoggerFactory.getLogger(TrackedBranchChainsProcessor.class);
+
+ /** Slow tracked branch operation threshold. */
+ private static final long SLOW_TRACKED_BRANCH_WARN_MS =
+ Long.getLong("tcbot.tracked.slowOperationWarnMs", 1000L);
+
/** TC ignited server provider. */
@Inject private ITeamcityIgnitedProvider tcIgnitedProv;
@@ -95,6 +105,15 @@ public class TrackedBranchChainsProcessor implements
IDetailedStatusForTrackedBr
int maxDurationSec,
boolean showMuted,
boolean showIgnored) {
+ long startNanos = System.nanoTime();
+ long chainTotalNanos = 0;
+ long serverResolveNanos = 0;
+ long tagFilterNanos = 0;
+ long historyNanos = 0;
+ long chainContextNanos = 0;
+ long uiInitNanos = 0;
+ long countersNanos;
+
final DsSummaryUi res = new DsSummaryUi();
final String branchNn = isNullOrEmpty(branch) ?
ITcServerConfig.DEFAULT_TRACKED_BRANCH_NAME : branch;
@@ -102,72 +121,106 @@ public class TrackedBranchChainsProcessor implements
IDetailedStatusForTrackedBr
final ITrackedBranch tracked =
tcBotCfg.getTrackedBranches().getBranchMandatory(branchNn);
- tracked.chainsStream()
+ List<ITrackedChain> accessibleChains = tracked.chainsStream()
.filter(chainTracked ->
tcIgnitedProv.hasAccess(chainTracked.serverCode(), creds))
- .map(chainTracked -> {
- final String srvCodeOrAlias = chainTracked.serverCode();
-
- final String branchForTc = chainTracked.tcBranch();
-
- //branch is tracked, so fail rate should be taken from this
branch data (otherwise it is specified).
- final String baseBranchTc =
chainTracked.tcBaseBranch().orElse(branchForTc);
-
- ITeamcityIgnited tcIgnited =
tcIgnitedProv.server(srvCodeOrAlias, creds);
-
- Map<Integer, Integer> requireParamVal = new HashMap<>();
-
- if (!Strings.isNullOrEmpty(tagForHistSelected)) {
- requireParamVal.putAll(
- reverseTagToParametersRequired(tagForHistSelected,
srvCodeOrAlias));
- }
+ .collect(Collectors.toList());
- DsChainUi chainStatus = new DsChainUi(srvCodeOrAlias,
- tcIgnited.serverCode(),
- branchForTc);
+ for (ITrackedChain chainTracked : accessibleChains) {
+ long chainStart = System.nanoTime();
+ final String srvCodeOrAlias = chainTracked.serverCode();
- chainStatus.baseBranchForTc = baseBranchTc;
+ final String branchForTc = chainTracked.tcBranch();
- String suiteIdMandatory = chainTracked.tcSuiteId();
+ //branch is tracked, so fail rate should be taken from this branch
data (otherwise it is specified).
+ final String baseBranchTc =
chainTracked.tcBaseBranch().orElse(branchForTc);
- List<Integer> chains =
tcIgnited.getLastNBuildsFromHistory(suiteIdMandatory, branchForTc,
buildResMergeCnt);
+ long stepStart = System.nanoTime();
+ ITeamcityIgnited tcIgnited = tcIgnitedProv.server(srvCodeOrAlias,
creds);
+ serverResolveNanos += System.nanoTime() - stepStart;
- ProcessLogsMode logs;
- if (buildResMergeCnt > 1)
- logs = (checkAllLogs != null && checkAllLogs) ?
ProcessLogsMode.ALL : ProcessLogsMode.DISABLED;
- else
- logs = (checkAllLogs != null && checkAllLogs) ?
ProcessLogsMode.ALL : ProcessLogsMode.SUITE_NOT_COMPLETE;
+ Map<Integer, Integer> requireParamVal = new HashMap<>();
- LatestRebuildMode rebuild = buildResMergeCnt > 1 ?
LatestRebuildMode.ALL : LatestRebuildMode.LATEST;
-
- boolean includeScheduled = buildResMergeCnt == 1;
-
- final FullChainRunCtx ctx = chainProc.loadFullChainContext(
- tcIgnited,
- chains,
- rebuild,
- logs,
- includeScheduled,
- baseBranchTc,
- syncMode,
- sortOption,
- requireParamVal
- );
-
- chainStatus.initFromContext(tcIgnited, ctx, baseBranchTc,
compactor, calcTrustedTests, tagSelected,
- displayMode, maxDurationSec, requireParamVal,
- showMuted, showIgnored);
+ if (!Strings.isNullOrEmpty(tagForHistSelected)) {
+ stepStart = System.nanoTime();
+ requireParamVal.putAll(
+ reverseTagToParametersRequired(tagForHistSelected,
srvCodeOrAlias));
+ tagFilterNanos += System.nanoTime() - stepStart;
+ }
- return chainStatus;
- })
- .forEach(res::addChainOnServer);
+ DsChainUi chainStatus = new DsChainUi(srvCodeOrAlias,
+ tcIgnited.serverCode(),
+ branchForTc);
+
+ chainStatus.baseBranchForTc = baseBranchTc;
+
+ String suiteIdMandatory = chainTracked.tcSuiteId();
+
+ stepStart = System.nanoTime();
+ List<Integer> chains =
tcIgnited.getLastNBuildsFromHistory(suiteIdMandatory, branchForTc,
buildResMergeCnt);
+ historyNanos += System.nanoTime() - stepStart;
+
+ ProcessLogsMode logs;
+ if (buildResMergeCnt > 1)
+ logs = (checkAllLogs != null && checkAllLogs) ?
ProcessLogsMode.ALL : ProcessLogsMode.DISABLED;
+ else
+ logs = (checkAllLogs != null && checkAllLogs) ?
ProcessLogsMode.ALL : ProcessLogsMode.SUITE_NOT_COMPLETE;
+
+ LatestRebuildMode rebuild = buildResMergeCnt > 1 ?
LatestRebuildMode.ALL : LatestRebuildMode.LATEST;
+
+ boolean includeScheduled = buildResMergeCnt == 1;
+
+ stepStart = System.nanoTime();
+ final FullChainRunCtx ctx = chainProc.loadFullChainContext(
+ tcIgnited,
+ chains,
+ rebuild,
+ logs,
+ includeScheduled,
+ baseBranchTc,
+ syncMode,
+ sortOption,
+ requireParamVal
+ );
+ chainContextNanos += System.nanoTime() - stepStart;
+
+ stepStart = System.nanoTime();
+ chainStatus.initFromContext(tcIgnited, ctx, baseBranchTc,
compactor, calcTrustedTests, tagSelected,
+ displayMode, maxDurationSec, requireParamVal,
+ showMuted, showIgnored);
+ uiInitNanos += System.nanoTime() - stepStart;
+
+ res.addChainOnServer(chainStatus);
+ chainTotalNanos += System.nanoTime() - chainStart;
+ }
res.servers.sort(Comparator.comparing(DsChainUi::serverName));
+ long stepStart = System.nanoTime();
res.initCounters(getTrackedBranchUpdateCounters(branch, creds));
+ countersNanos = System.nanoTime() - stepStart;
+
+ long totalMs = nanosToMillis(System.nanoTime() - startNanos);
+
+ if (totalMs >= SLOW_TRACKED_BRANCH_WARN_MS) {
+ logger.warn("Slow tracked branch budget: branch={}, chains={},
count={}, syncMode={}, totalMs={}, " +
+ "chainTotalMs={}, serverResolveMs={}, tagFilterMs={},
historyMs={}, chainContextMs={}, " +
+ "uiInitMs={}, countersMs={}",
+ branchNn, accessibleChains.size(), buildResMergeCnt, syncMode,
totalMs,
+ nanosToMillis(chainTotalNanos),
nanosToMillis(serverResolveNanos), nanosToMillis(tagFilterNanos),
+ nanosToMillis(historyNanos), nanosToMillis(chainContextNanos),
nanosToMillis(uiInitNanos),
+ nanosToMillis(countersNanos));
+ }
return res;
}
+ /**
+ * @param nanos Nanoseconds.
+ */
+ private static long nanosToMillis(long nanos) {
+ return TimeUnit.NANOSECONDS.toMillis(nanos);
+ }
+
public Map<Integer, Integer> reverseTagToParametersRequired(@Nullable
String tagForHistSelected,
String srvCodeOrAlias) {
