This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch ignite-10336 in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-10336 by this push: new 8cee0f4 IGNITE-10336 Chain collection refactoring 8cee0f4 is described below commit 8cee0f4f66ddf781dfeb318b625080f41c481fde Author: Dmitriy Pavlov <dpav...@apache.org> AuthorDate: Wed Nov 21 17:21:51 2018 +0300 IGNITE-10336 Chain collection refactoring --- .../apache/ignite/ci/analysis/FullChainRunCtx.java | 2 +- .../ignite/ci/analysis/mode/LatestRebuildMode.java | 2 +- .../ignite/ci/tcbot/chain/BuildChainProcessor.java | 166 ++++++++++++--------- .../ci/teamcity/ignited/BuildRefCompacted.java | 7 +- .../ignited/fatbuild/FatBuildCompacted.java | 3 + .../ci/tcbot/chain/BuildChainProcessorTest.java | 2 +- 6 files changed, 107 insertions(+), 75 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java index 36602c6..c9cf274 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/FullChainRunCtx.java @@ -90,7 +90,7 @@ public class FullChainRunCtx { + (hasFullDurationInfo() ? "" : "+"); } - public void addAllSuites(ArrayList<MultBuildRunCtx> suites) { + public void addAllSuites(List<MultBuildRunCtx> suites) { this.buildCfgsResults.addAll(suites); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/mode/LatestRebuildMode.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/mode/LatestRebuildMode.java index 5083bb7..9583f5a 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/mode/LatestRebuildMode.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/analysis/mode/LatestRebuildMode.java @@ -22,6 +22,6 @@ public enum LatestRebuildMode { NONE, /** replace builds with Latest rebuild. */ LATEST, - /** Collect history of builds. */ + /** Collect history of builds. Rebuilds are applied, but have higher priority. */ ALL } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java index 66cfef8..d1b0c89 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java @@ -27,6 +27,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -95,25 +96,9 @@ public class BuildChainProcessor { if (entryPoints.isEmpty()) return res; - Map<Integer, FatBuildCompacted> builds = new ConcurrentHashMap<>(); + Map<Integer, Future<FatBuildCompacted>> builds = loadChain(entryPoints, mode, teamcityIgnited); - final Stream<FatBuildCompacted> entryPointsFatBuilds = entryPoints.stream() - .filter(Objects::nonNull) - .filter(id -> !builds.containsKey(id)) //load and propagate only new entry points - .map(id -> builds.computeIfAbsent(id, teamcityIgnited::getFatBuild)); - - final ExecutorService svc = tcUpdatePool.getService(); - - final Stream<FatBuildCompacted> depsFirstLevel = - Stream.of(); //todo implement - /*entryPointsFatBuilds - .map(ref -> svc.submit(() -> dependencies(teamcityIgnited, builds, mode, ref))) - .flatMap(set->set.stream()) - .stream() - .flatMap(fut -> FutureUtil.getResult(fut)); - -*/ - depsFirstLevel + builds.values().stream().map(FutureUtil::getResult) .filter(b -> !b.isComposite() && b.getTestsCount() > 0) .forEach(b -> { @@ -184,41 +169,47 @@ public class BuildChainProcessor { if (entryPoints.isEmpty()) return new FullChainRunCtx(Build.createFakeStub()); - Map<Integer, Future<FatBuildCompacted>> builds = new ConcurrentHashMap<>(); + Map<Integer, Future<FatBuildCompacted>> builds = loadChain(entryPoints, mode, tcIgn); - Stream<Future<FatBuildCompacted>> entryPointsFatBuilds = entryPoints.stream() - .filter(Objects::nonNull) - .map(id -> builds.computeIfAbsent(id, id0 -> loadBuildAsync(id0, mode, tcIgn))); + Map<String, List<Future<FatBuildCompacted>>> freshRebuilds = new ConcurrentHashMap<>(); - Stream<Integer> dependenciesFirstLevel = entryPointsFatBuilds - .flatMap(ref -> dependencies(ref, mode, builds, tcIgn).stream()); + groupByBuildType(builds).forEach( + (k, buildsForBt) -> { + List<Future<FatBuildCompacted>> futures = replaceWithRecent(buildsForBt, + entryPoints.size(), + includeLatestRebuild, + builds, + mode, + tcIgn); - Stream<Integer> depsSecondLevel = dependenciesFirstLevel.map(builds::get) - .peek(val -> Preconditions.checkNotNull(val, "Build future should be in context")) - .flatMap(ref -> dependencies(ref, mode, builds, tcIgn).stream()); + freshRebuilds.put(k, futures); + } + ); - Set<Integer> collect = depsSecondLevel.collect(Collectors.toSet()); - System.err.println("New deps of second level:" + collect); + List<MultBuildRunCtx> contexts = new ArrayList<>(freshRebuilds.size()); - // builds may became non unique because of race in filtering and acquiring deps - //todo implement - /* final List<Future<Stream<FatBuildCompacted>>> phase3Submitted = secondLevelDeps - .map((fatBuild) -> svc.submit( - () -> replaceWithRecent(tcIgn, includeLatestRebuild, mode, builds, fatBuild, entryPoints.size()))) - .collect(Collectors.toList());*/ + freshRebuilds.forEach((bt, listBuilds) -> { + List<FatBuildCompacted> buildsForSuite = listBuilds.stream() + .map(FutureUtil::getResult) + .filter(buildCompacted -> { + return !buildCompacted.isFakeStub() && buildCompacted.getId()!=null; + }) + .collect(Collectors.toList()); - Map<String, MultBuildRunCtx> buildsByBt = new ConcurrentHashMap<>(); + if (buildsForSuite.isEmpty()) + return; - builds.values().stream() - .map(FutureUtil::getResult) - .forEach((fatBuild) -> createCxt(tcIgn, buildsByBt, fatBuild)); + BuildRef ref = buildsForSuite.iterator().next().toBuildRef(compactor); + + final MultBuildRunCtx ctx = new MultBuildRunCtx(ref, compactor); - ArrayList<MultBuildRunCtx> contexts = new ArrayList<>(buildsByBt.values()); + buildsForSuite.forEach(buildCompacted -> ctx.addBuild(loadChanges(buildCompacted, tcIgn))); - contexts.forEach(multiCtx -> { - analyzeTests(multiCtx, teamcity, procLog); + analyzeTests(ctx, teamcity, procLog); - fillBuildCounts(multiCtx, tcIgn, includeScheduledInfo); + fillBuildCounts(ctx, tcIgn, includeScheduledInfo); + + contexts.add(ctx); }); Function<MultBuildRunCtx, Float> function = ctx -> { @@ -235,7 +226,7 @@ public class BuildChainProcessor { }; Integer someEntryPnt = entryPoints.iterator().next(); - Future<FatBuildCompacted> build = builds.computeIfAbsent(someEntryPnt, id -> loadBuildAsync(id, mode, tcIgn)); + Future<FatBuildCompacted> build = getOrLoadBuild(someEntryPnt, mode, builds, tcIgn); FullChainRunCtx fullChainRunCtx = new FullChainRunCtx(FutureUtil.getResult(build).toBuild(compactor)); contexts.sort(Comparator.comparing(function).reversed()); @@ -245,21 +236,42 @@ public class BuildChainProcessor { return fullChainRunCtx; } + @NotNull + public Map<Integer, Future<FatBuildCompacted>> loadChain(Collection<Integer> entryPoints, + SyncMode mode, + ITeamcityIgnited tcIgn) { + Map<Integer, Future<FatBuildCompacted>> builds = new ConcurrentHashMap<>(); - @SuppressWarnings("WeakerAccess") - @AutoProfiling - protected void createCxt(ITeamcityIgnited teamcityIgnited, - Map<String, MultBuildRunCtx> buildsCtxMap, - FatBuildCompacted buildCompacted) { - final BuildRef ref = buildCompacted.toBuildRef(compactor); + Stream<Future<FatBuildCompacted>> entryPointsFatBuilds = entryPoints.stream() + .filter(Objects::nonNull) + .map(id -> getOrLoadBuild(id, mode, builds, tcIgn)); - if (buildCompacted.isFakeStub() || ref.isFakeStub()) - return; + Stream<Integer> dependenciesFirstLevel = entryPointsFatBuilds + .flatMap(ref -> dependencies(ref, mode, builds, tcIgn).stream()); - final MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(ref.buildTypeId, - k -> new MultBuildRunCtx(ref, compactor)); + Stream<Integer> depsSecondLevel = dependenciesFirstLevel.map(builds::get) + .peek(val -> Preconditions.checkNotNull(val, "Build future should be in context")) + .flatMap(ref -> dependencies(ref, mode, builds, tcIgn).stream()); - ctx.addBuild(loadChanges(buildCompacted, teamcityIgnited)); + Set<Integer> collect = depsSecondLevel.collect(Collectors.toSet()); + System.err.println("New deps of second level:" + collect); + return builds; + } + + @NotNull + public Map<String, List<FatBuildCompacted>> groupByBuildType(Map<Integer, Future<FatBuildCompacted>> builds) { + Map<String, List<FatBuildCompacted>> buildsByBt = new ConcurrentHashMap<>(); + builds.values().forEach(bFut -> { + FatBuildCompacted b = FutureUtil.getResult(bFut); + + buildsByBt.computeIfAbsent(b.buildTypeId(compactor), k -> new ArrayList<>()).add(b); + }); + return buildsByBt; + } + + public Future<FatBuildCompacted> getOrLoadBuild(Integer id, SyncMode mode, + Map<Integer, Future<FatBuildCompacted>> builds, ITeamcityIgnited tcIgn) { + return builds.computeIfAbsent(id, id0 -> loadBuildAsync(id0, mode, tcIgn)); } /** @@ -281,37 +293,43 @@ public class BuildChainProcessor { @SuppressWarnings("WeakerAccess") @NotNull @AutoProfiling - protected Stream<FatBuildCompacted> replaceWithRecent(ITeamcityIgnited teamcityIgnited, + protected List<Future<FatBuildCompacted>> replaceWithRecent(List<FatBuildCompacted> builds, + int cntLimit, LatestRebuildMode includeLatestRebuild, + Map<Integer, Future<FatBuildCompacted>> allBuildsMap, SyncMode syncMode, - Map<Integer, FatBuildCompacted> builds, - FatBuildCompacted buildCompacted, - int cntLimit) { - if (includeLatestRebuild == LatestRebuildMode.NONE) - return Stream.of(buildCompacted); + ITeamcityIgnited tcIgn) { + if (includeLatestRebuild == LatestRebuildMode.NONE || builds.isEmpty()) + return completed(builds); + + Optional<FatBuildCompacted> maxIdBuildOpt = builds.stream().max(Comparator.comparing(BuildRefCompacted::id)); + if (!maxIdBuildOpt.isPresent()) + return completed(builds); - final String branch = getBranchOrDefault(buildCompacted.branchName(compactor)); + FatBuildCompacted freshBuild = maxIdBuildOpt.get(); - final String buildTypeId = buildCompacted.buildTypeId(compactor); - Stream<BuildRefCompacted> hist = teamcityIgnited.getAllBuildsCompacted(buildTypeId, branch) + final String branch = getBranchOrDefault(freshBuild.branchName(compactor)); + + final String buildTypeId = freshBuild.buildTypeId(compactor); + Stream<BuildRefCompacted> hist = tcIgn.getAllBuildsCompacted(buildTypeId, branch) .stream() - .filter(t -> !t.isCancelled(compactor)) - .filter(t -> t.isFinished(compactor)); + .filter(bref -> !bref.isCancelled(compactor)) + .filter(bref -> bref.isFinished(compactor)); if (includeLatestRebuild == LatestRebuildMode.LATEST) { BuildRefCompacted recentRef = hist.max(Comparator.comparing(BuildRefCompacted::id)) - .orElse(buildCompacted); + .orElse(freshBuild); - return Stream.of(recentRef) - .map(b -> builds.computeIfAbsent(b.id(), id -> FutureUtil.getResult(loadBuildAsync(id, syncMode, teamcityIgnited)))); + return Collections.singletonList( + getOrLoadBuild(recentRef.id(), syncMode, allBuildsMap, tcIgn)); } if (includeLatestRebuild == LatestRebuildMode.ALL) { return hist .sorted(Comparator.comparing(BuildRefCompacted::id).reversed()) .limit(cntLimit) - // .filter(b -> !builds.containsKey(b.id())) // todo removing this causes incorrect count of failures (duplicated builds) - .map(b -> builds.computeIfAbsent(b.id(), id -> FutureUtil.getResult(loadBuildAsync(id, syncMode, teamcityIgnited)))); + .map(bref -> getOrLoadBuild(bref.id(), syncMode, allBuildsMap, tcIgn)) + .collect(Collectors.toList()); } throw new UnsupportedOperationException("invalid mode " + includeLatestRebuild); @@ -405,4 +423,10 @@ public class BuildChainProcessor { return teamcityIgnited.getFatBuild(id, mode); }); } + + + private List<Future<FatBuildCompacted>> completed(List<FatBuildCompacted> builds) { + return builds.stream().map(Futures::immediateFuture).collect(Collectors.toList()); + } + } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java index efda317..f6ccf03 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/BuildRefCompacted.java @@ -22,6 +22,7 @@ import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.ci.db.Persisted; import org.apache.ignite.ci.tcmodel.hist.BuildRef; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import static org.apache.ignite.ci.tcmodel.hist.BuildRef.*; @@ -93,7 +94,7 @@ public class BuildRefCompacted { } protected void fillBuildRefFields(IStringCompactor compactor, BuildRef res) { - res.setId(id < 0 ? null : id); + res.setId(getId()); res.buildTypeId = buildTypeId(compactor); res.branchName = branchName(compactor); res.status = compactor.getStringFromId(status); @@ -101,6 +102,10 @@ public class BuildRefCompacted { res.href = getHrefForId(id()); } + @Nullable public Integer getId() { + return id < 0 ? null : id; + } + public String buildTypeId(IStringCompactor compactor) { return compactor.getStringFromId(buildTypeId); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java index 95e88ad..419de92 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java @@ -330,6 +330,9 @@ public class FatBuildCompacted extends BuildRefCompacted implements IVersionedEn * */ public boolean isFakeStub() { + if (getId() == null) + return true; + Boolean flag = getFlag(FAKE_BUILD_F); return flag != null && flag; diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java index 207f852..23c735e 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java @@ -141,7 +141,7 @@ public class BuildChainProcessorTest { if (suite.suiteName() != null && suite.suiteName().startsWith(UNIQUE_FAILED_TEST)) { for (ITestFailures test : suite.getFailedTests()) - assertTrue("Failure found but should be hidden by re-run " + test, false); + assertTrue("Failure found but should be hidden by re-run " + test.getName(), false); } } }