This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push: new 3f55641 Tracked branch list now contains number of suites running and queued 3f55641 is described below commit 3f55641bb2aabc0e4f77455ede46c4efa664c9f7 Author: Dmitriy Pavlov <dpav...@apache.org> AuthorDate: Thu Aug 1 18:05:11 2019 +0300 Tracked branch list now contains number of suites running and queued --- .../org/apache/ignite/ci/web/model/Version.java | 2 +- .../rest/tracked/GetTrackedBranchTestResults.java | 17 ++++++ ignite-tc-helper-web/src/main/webapp/guard.html | 35 +++++++++++++ .../tracked/IDetailedStatusForTrackedBranch.java | 7 +++ .../tracked/TrackedBranchChainsProcessor.java | 60 ++++++++++++++++++++++ .../tcbot/engine/ui/GuardBranchStatusUi.java | 58 +++++++++++++++++++++ .../apache/ignite/tcignited/ITeamcityIgnited.java | 4 ++ .../ignite/tcignited/TeamcityIgnitedImpl.java | 8 ++- .../ignite/tcignited/history/HistoryCollector.java | 44 ++++++++-------- 9 files changed, 210 insertions(+), 25 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java index b1ce2f9..a9170d3 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/Version.java @@ -28,7 +28,7 @@ package org.apache.ignite.ci.web.model; public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot"; /** TC Bot Version. */ - public static final String VERSION = "20190731"; + public static final String VERSION = "20190801"; /** Java version, where Web App is running. */ public String javaVer; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java index 1ff6d7c..3b87184 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java @@ -18,7 +18,10 @@ package org.apache.ignite.ci.web.rest.tracked; import com.google.inject.Injector; +import java.util.List; +import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; @@ -38,6 +41,7 @@ import org.apache.ignite.tcbot.engine.tracked.DisplayMode; import org.apache.ignite.tcbot.engine.tracked.IDetailedStatusForTrackedBranch; import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor; import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; +import org.apache.ignite.tcbot.engine.ui.GuardBranchStatusUi; import org.apache.ignite.tcbot.engine.ui.UpdateInfo; import org.apache.ignite.tcignited.ITeamcityIgnitedProvider; import org.apache.ignite.tcignited.SyncMode; @@ -220,4 +224,17 @@ public class GetTrackedBranchTestResults { .getInstance(TcBotTriggerAndSignOffService.class) .getMutes(srvCode, projectId, creds); } + + @GET + @Path("summary") + public List<GuardBranchStatusUi> getIdsIfAccessible() { + ITcBotUserCreds prov = ITcBotUserCreds.get(req); + Injector injector = CtxListener.getInjector(ctx); + ITcBotConfig cfg = injector.getInstance(ITcBotConfig.class); + IDetailedStatusForTrackedBranch status = injector.getInstance(IDetailedStatusForTrackedBranch.class); + + return cfg.getTrackedBranches().branchesStream() + .map(bt -> status.getBranchSummary(bt.name(), prov)).filter(Objects::nonNull) + .collect(Collectors.toList()); + } } diff --git a/ignite-tc-helper-web/src/main/webapp/guard.html b/ignite-tc-helper-web/src/main/webapp/guard.html index 0078687..8c8d303 100644 --- a/ignite-tc-helper-web/src/main/webapp/guard.html +++ b/ignite-tc-helper-web/src/main/webapp/guard.html @@ -33,15 +33,50 @@ function loadData() { success: function(result) { $("#loadStatus").html(""); showTrackedBranchesLinks(result); + + $.ajax({ + url: "rest/tracked/summary", + success: function(result) { + $("#loadStatus").html(""); + showTrackedBranchesLinks2(result); + }, + error: showErrInLoadStatus + }); }, error: showErrInLoadStatus }); + +} + +function showTrackedBranchesLinks2(result) { + var res = "<table >"; + res+="<tr><td>Branch name</td><td>Report</td><td>Merged 10</td><td>Finished last day</td><td>Running</td><td>Queued</td></tr>"; + for (var i = 0; i < result.length; i++) { + let summary = result[i]; + var id = summary.name; + res += "<tr>"; + res += "<td align='right'>" + id + ":</td>"; + res += "<td>" + "<a href='current.html?branch=" + id + "'><button>Latest (" + id + ")</button></a>" + "</td>"; + res += "<td>" + " <a href='current.html?branch=" + id + "&count=10'><button title='Failures from last 10 runs merged'>History (" + id + ")</button></a>" + "</td>"; + + res += "<td>" + JSON.stringify(summary.finishedLastDay)+ "</td>"; + res += "<td>" + JSON.stringify(summary.runningList)+ "</td>"; + res += "<td>" + JSON.stringify(summary.queuedList)+ "</td>"; + res += "</tr>"; + } + + res += "</table>"; + + + $("#branchesStatus").html(res); } function showTrackedBranchesLinks(result) { var res = "<table>"; + res+="<tr><td>Branch name</td><td>Report</td><td>Merged 10</td><td>Finished last day</td><td>Running</td><td>Queued</td></tr>"; + for (var i = 0; i < result.length; i++) { var id = result[i]; res += "<tr>"; diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java index 4edad4a..1ba1e7c 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java @@ -19,6 +19,7 @@ package org.apache.ignite.tcbot.engine.tracked; import javax.annotation.Nullable; import org.apache.ignite.tcbot.engine.chain.SortOption; import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; +import org.apache.ignite.tcbot.engine.ui.GuardBranchStatusUi; import org.apache.ignite.tcignited.SyncMode; import org.apache.ignite.tcignited.creds.ICredentialsProv; @@ -52,5 +53,11 @@ public interface IDetailedStatusForTrackedBranch { @Nullable SortOption sortOption, int maxDurationSec); + /** + * @param name Name. + * @param prov Prov. + */ + public GuardBranchStatusUi getBranchSummary(String name, ICredentialsProv prov); + // * @param baseTrackedBranch Branch tracked branch in Bot, has a priority if both TC & Bot branches (baseBranchForTcParm) present. } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java index 56d266a..0cbab4e 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java @@ -17,14 +17,17 @@ package org.apache.ignite.tcbot.engine.tracked; import com.google.common.base.Strings; +import java.time.Duration; import java.util.Collection; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.inject.Inject; +import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; import org.apache.ignite.tcbot.common.conf.IBuildParameterSpec; import org.apache.ignite.tcbot.common.conf.IParameterValueSpec; import org.apache.ignite.tcbot.common.conf.ITcServerConfig; @@ -36,8 +39,10 @@ import org.apache.ignite.tcbot.engine.chain.ProcessLogsMode; import org.apache.ignite.tcbot.engine.chain.SortOption; import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; import org.apache.ignite.tcbot.engine.conf.ITrackedBranch; +import org.apache.ignite.tcbot.engine.conf.ITrackedChain; import org.apache.ignite.tcbot.engine.ui.DsChainUi; import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; +import org.apache.ignite.tcbot.engine.ui.GuardBranchStatusUi; import org.apache.ignite.tcbot.engine.ui.LrTestsFullSummaryUi; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; @@ -166,6 +171,61 @@ public class TrackedBranchChainsProcessor implements IDetailedStatusForTrackedBr return res; } + @Override public GuardBranchStatusUi getBranchSummary(String name, ICredentialsProv prov) { + ITrackedBranch tb = tcBotCfg.getTrackedBranches().getBranchMandatory(name); + List<ITrackedChain> accessibleChains = + tb.chainsStream() + .filter(chain -> tcIgnitedProv.hasAccess(chain.serverCode(), prov)) + .collect(Collectors.toList()); + + if (accessibleChains == null) + return null; + + int ageDays = 1; + long minStartTime = System.currentTimeMillis() - Duration.ofDays(ageDays).toMillis(); + + GuardBranchStatusUi statusUi = new GuardBranchStatusUi(); + statusUi.setName(tb.name()); + + for (ITrackedChain chain : accessibleChains) { + String srvCodeOrAlias = chain.serverCode(); + ITeamcityIgnited tcIgn = tcIgnitedProv.server(srvCodeOrAlias, prov); + + List<BuildRefCompacted> hist = tcIgn.getAllBuildsCompacted(chain.tcSuiteId(), chain.tcBranch()); + + AtomicInteger finished = new AtomicInteger(); + AtomicInteger running = new AtomicInteger(); + AtomicInteger queued = new AtomicInteger(); + + hist.stream() + .filter(ref -> !ref.isFakeStub()) + .filter(t -> !t.isCancelled(compactor)) + .peek(ref -> { + if (ref.isRunning(compactor)) + running.incrementAndGet(); + else if (ref.isQueued(compactor)) + queued.incrementAndGet(); + }) + .filter(ref -> ref.isFinished(compactor)) + .filter(ref -> { + Integer borderId = tcIgn.getBorderForAgeForBuildId(ageDays); + return borderId == null || ref.id() >= borderId; + }) + .filter(ref -> { + Long startTime = tcIgn.getBuildStartTime(ref.id()); + + return startTime != null && startTime > minStartTime; + }) + .forEach(ref -> { + finished.incrementAndGet(); + }); + + statusUi.addSuiteRunStat(finished.get(), running.get(), queued.get()); + } + + return statusUi; + } + /** * Collects data about all long-running tests (run time more than one minute) within one transfer object. * diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/GuardBranchStatusUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/GuardBranchStatusUi.java new file mode 100644 index 0000000..fd3378a --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/GuardBranchStatusUi.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.tcbot.engine.ui; + +import java.util.ArrayList; +import java.util.List; + +/** + * Branch summary to be shown at guard page, listing + */ +public class GuardBranchStatusUi { + private String name; + + private List<Integer> finishedLastDay = new ArrayList<>(); + private List<Integer> runningList = new ArrayList<>(); + private List<Integer> queuedList = new ArrayList<>(); + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List<Integer> getFinishedLastDay() { + return finishedLastDay; + } + + public void addSuiteRunStat(int finished, int running, int queued) { + finishedLastDay.add(finished); + runningList.add(running); + queuedList.add(queued); + } + + + public List<Integer> getRunningList() { + return runningList; + } + + public List<Integer> getQueuedList() { + return queuedList; + } +} diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java index 17f82bf..e68fd46 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/ITeamcityIgnited.java @@ -259,4 +259,8 @@ public interface ITeamcityIgnited { * should be re-synced. */ public void actualizeRecentBuildRefs(); + + public Long getBuildStartTime(int buildId); + + public Integer getBorderForAgeForBuildId(int days); } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java index 6bdb464..c7b20b7 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/TeamcityIgnitedImpl.java @@ -16,7 +16,6 @@ */ package org.apache.ignite.tcignited; -import com.google.common.collect.Sets; import java.io.File; import java.util.ArrayList; import java.util.Collection; @@ -554,12 +553,17 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { return null; } - //@GuavaCached(maximumSize = 100000, expireAfterAccessSecs = 90, softValues = true) + /** {@inheritDoc} */ public Long getBuildStartTime(int buildId) { return histCollector.getBuildStartTime(srvIdMaskHigh, buildId); } /** {@inheritDoc} */ + @Override public Integer getBorderForAgeForBuildId(int days) { + return buildStartTimeStorage.getBorderForAgeForBuildId(srvIdMaskHigh, days); + } + + /** {@inheritDoc} */ @GuavaCached(maximumSize = 500, expireAfterAccessSecs = 30, softValues = true) @Override public FatBuildCompacted getFatBuild(int buildId, SyncMode mode) { FatBuildCompacted existingBuild = getFatBuildFromIgnite(buildId); diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java index ab93519..5785a4d 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/HistoryCollector.java @@ -18,6 +18,25 @@ package org.apache.ignite.tcignited.history; import com.google.common.cache.CacheBuilder; import com.google.common.collect.Iterables; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiPredicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.cache.Cache; +import javax.inject.Inject; +import javax.inject.Provider; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.binary.BinaryObject; @@ -42,20 +61,6 @@ import org.apache.ignite.tcservice.model.result.tests.TestOccurrence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.cache.Cache; -import javax.inject.Inject; -import javax.inject.Provider; -import java.time.Duration; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiPredicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * */ @@ -147,24 +152,19 @@ public class HistoryCollector { long curTs = System.currentTimeMillis(); Set<Integer> buildIds = bRefsList.stream() .filter(b -> { - /* AtomicInteger biggestIdOutOfScope = biggestBuildIdOutOfHistoryScope.get(srvId); - int outOfScopeBuildId = biggestIdOutOfScope == null ? -1 : biggestIdOutOfScope.get(); - return b.id() > outOfScopeBuildId; - */ - Integer maxBuildIdForDay = buildStartTimeStorage.getBorderForAgeForBuildId(srvId, TcBotConst.HISTORY_BUILD_ID_BORDER_DAYS); if (maxBuildIdForDay == null) return true; - return b.id()>maxBuildIdForDay; + return b.id() > maxBuildIdForDay; }) .filter(this::applicableForHistory) .map(BuildRefCompacted::id) .filter(bId -> !knownBuilds.contains(bId)).collect(Collectors.toSet()); - System.out.println("***** Loading build start time history for suite " + logger.info("***** Loading build start time history for suite " + compactor.getStringFromId(buildTypeId) + " branch " + compactor.getStringFromId(normalizedBaseBranch) + ": " + buildIds.size() + " builds" ); @@ -207,7 +207,7 @@ public class HistoryCollector { } ).collect(Collectors.toSet()); - System.err.println("*** Build " + btId + " branch " + branchId + " builds in scope " + + logger.info("*** Build " + btId + " branch " + branchId + " builds in scope " + buildInScope.size() + " from " + bRefsList.size()); return buildInScope;