This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch ignite-10336 in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 5d3aad65544903f6d77afc222bf8160cce8afd2a Author: Dmitriy Pavlov <dpav...@apache.org> AuthorDate: Wed Nov 21 15:33:06 2018 +0300 IGNITE-10336 Chain collection refactoring started --- .../ci/tcbot/builds/CompareBuildsService.java | 2 +- .../ignite/ci/tcbot/chain/BuildChainProcessor.java | 131 ++++++++++----------- .../ci/tcbot/chain/BuildChainProcessorTest.java | 44 ++++++- 3 files changed, 104 insertions(+), 73 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java index 71715c8..e420ca4 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java @@ -80,7 +80,7 @@ public class CompareBuildsService { MultBuildRunCtx buildCtx = new MultBuildRunCtx(build, compactor); final FatBuildCompacted fatBuild = tcIgnited.getFatBuild(build.getId()); - buildCtx.addBuild(bcp.loadTestsAndProblems(fatBuild, tcIgnited)); + buildCtx.addBuild(bcp.loadChanges(fatBuild, tcIgnited)); for (String testName : buildCtx.tests()) tests.add(extractTestName(testName)); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java index 14d6d2e..66cfef8 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java @@ -17,14 +17,17 @@ package org.apache.ignite.ci.tcbot.chain; +import com.google.common.base.Preconditions; import com.google.common.util.concurrent.Futures; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -101,12 +104,15 @@ public class BuildChainProcessor { final ExecutorService svc = tcUpdatePool.getService(); - final Stream<FatBuildCompacted> depsFirstLevel = entryPointsFatBuilds + final Stream<FatBuildCompacted> depsFirstLevel = + Stream.of(); //todo implement + /*entryPointsFatBuilds .map(ref -> svc.submit(() -> dependencies(teamcityIgnited, builds, mode, ref))) - .collect(Collectors.toList()) + .flatMap(set->set.stream()) .stream() .flatMap(fut -> FutureUtil.getResult(fut)); +*/ depsFirstLevel .filter(b -> !b.isComposite() && b.getTestsCount() > 0) .forEach(b -> @@ -121,7 +127,7 @@ public class BuildChainProcessor { ); if (!lrTests.isEmpty()) { - Collections.sort(lrTests, (test0, test1) -> { + lrTests.sort((test0, test1) -> { long t0 = test0.time; long t1 = test1.time; @@ -156,18 +162,18 @@ public class BuildChainProcessor { /** * @param teamcity Teamcity. - * @param teamcityIgnited + * @param tcIgn Teamcity Ignited. * @param entryPoints Entry point(s): Build(s) to start scan from. * @param includeLatestRebuild Include latest rebuild. * @param procLog Process logger. * @param includeScheduledInfo Include scheduled info. * @param failRateBranch Fail rate branch. - * @param mode + * @param mode background data update mode. */ @AutoProfiling public FullChainRunCtx loadFullChainContext( IAnalyticsEnabledTeamcity teamcity, - ITeamcityIgnited teamcityIgnited, + ITeamcityIgnited tcIgn, Collection<Integer> entryPoints, LatestRebuildMode includeLatestRebuild, ProcessLogsMode procLog, @@ -178,39 +184,41 @@ public class BuildChainProcessor { if (entryPoints.isEmpty()) return new FullChainRunCtx(Build.createFakeStub()); - Map<Integer, FatBuildCompacted> builds = new ConcurrentHashMap<>(); + Map<Integer, Future<FatBuildCompacted>> builds = new ConcurrentHashMap<>(); - final Stream<FatBuildCompacted> entryPointsFatBuilds = entryPoints.stream() - .filter(Objects::nonNull) - .filter(id -> !builds.containsKey(id)) //load and propagate only new entry points - .map(id -> builds.computeIfAbsent(id, id0 -> teamcityIgnited.getFatBuild(id0, mode))); + Stream<Future<FatBuildCompacted>> entryPointsFatBuilds = entryPoints.stream() + .filter(Objects::nonNull) + .map(id -> builds.computeIfAbsent(id, id0 -> loadBuildAsync(id0, mode, tcIgn))); - final ExecutorService svc = tcUpdatePool.getService(); + Stream<Integer> dependenciesFirstLevel = entryPointsFatBuilds + .flatMap(ref -> dependencies(ref, mode, builds, tcIgn).stream()); - final Stream<FatBuildCompacted> depsFirstLevel = entryPointsFatBuilds - .flatMap(ref -> dependencies(teamcityIgnited, builds, mode, ref)); + Stream<Integer> depsSecondLevel = dependenciesFirstLevel.map(builds::get) + .peek(val -> Preconditions.checkNotNull(val, "Build future should be in context")) + .flatMap(ref -> dependencies(ref, mode, builds, tcIgn).stream()); - Stream<FatBuildCompacted> secondLevelDeps = depsFirstLevel - .flatMap(ref -> dependencies(teamcityIgnited, builds, mode, ref)); + Set<Integer> collect = depsSecondLevel.collect(Collectors.toSet()); + System.err.println("New deps of second level:" + collect); // builds may became non unique because of race in filtering and acquiring deps - final List<Future<Stream<FatBuildCompacted>>> phase3Submitted = secondLevelDeps + //todo implement + /* final List<Future<Stream<FatBuildCompacted>>> phase3Submitted = secondLevelDeps .map((fatBuild) -> svc.submit( - () -> replaceWithRecent(teamcityIgnited, includeLatestRebuild, mode, builds, fatBuild, entryPoints.size()))) - .collect(Collectors.toList()); + () -> replaceWithRecent(tcIgn, includeLatestRebuild, mode, builds, fatBuild, entryPoints.size()))) + .collect(Collectors.toList());*/ - Map<String, MultBuildRunCtx> buildsCtxMap = new ConcurrentHashMap<>(); + Map<String, MultBuildRunCtx> buildsByBt = new ConcurrentHashMap<>(); - phase3Submitted.stream() - .flatMap(fut -> FutureUtil.getResult(fut)) - .forEach((fatBuild) -> createCxt(teamcityIgnited, buildsCtxMap, fatBuild)); + builds.values().stream() + .map(FutureUtil::getResult) + .forEach((fatBuild) -> createCxt(tcIgn, buildsByBt, fatBuild)); - ArrayList<MultBuildRunCtx> contexts = new ArrayList<>(buildsCtxMap.values()); + ArrayList<MultBuildRunCtx> contexts = new ArrayList<>(buildsByBt.values()); contexts.forEach(multiCtx -> { analyzeTests(multiCtx, teamcity, procLog); - fillBuildCounts(multiCtx, teamcityIgnited, includeScheduledInfo); + fillBuildCounts(multiCtx, tcIgn, includeScheduledInfo); }); Function<MultBuildRunCtx, Float> function = ctx -> { @@ -227,8 +235,8 @@ public class BuildChainProcessor { }; Integer someEntryPnt = entryPoints.iterator().next(); - FatBuildCompacted build = builds.computeIfAbsent(someEntryPnt, id -> teamcityIgnited.getFatBuild(id, mode)); - FullChainRunCtx fullChainRunCtx = new FullChainRunCtx(build.toBuild(compactor)); + Future<FatBuildCompacted> build = builds.computeIfAbsent(someEntryPnt, id -> loadBuildAsync(id, mode, tcIgn)); + FullChainRunCtx fullChainRunCtx = new FullChainRunCtx(FutureUtil.getResult(build).toBuild(compactor)); contexts.sort(Comparator.comparing(function).reversed()); @@ -237,6 +245,7 @@ public class BuildChainProcessor { return fullChainRunCtx; } + @SuppressWarnings("WeakerAccess") @AutoProfiling protected void createCxt(ITeamcityIgnited teamcityIgnited, @@ -250,7 +259,7 @@ public class BuildChainProcessor { final MultBuildRunCtx ctx = buildsCtxMap.computeIfAbsent(ref.buildTypeId, k -> new MultBuildRunCtx(ref, compactor)); - ctx.addBuild(loadTestsAndProblems(buildCompacted, teamcityIgnited)); + ctx.addBuild(loadChanges(buildCompacted, teamcityIgnited)); } /** @@ -260,14 +269,12 @@ public class BuildChainProcessor { * @param tcIgnited * @return Full context. */ - public SingleBuildRunCtx loadTestsAndProblems(@Nonnull FatBuildCompacted buildCompacted, + public SingleBuildRunCtx loadChanges(@Nonnull FatBuildCompacted buildCompacted, ITeamcityIgnited tcIgnited) { SingleBuildRunCtx ctx = new SingleBuildRunCtx(buildCompacted, compactor); ctx.setChanges(tcIgnited.getAllChanges(buildCompacted.changes())); - //todo support storing build.lastChanges.changes) ? - return ctx; } @@ -296,7 +303,7 @@ public class BuildChainProcessor { .orElse(buildCompacted); return Stream.of(recentRef) - .map(b -> builds.computeIfAbsent(b.id(), id -> teamcityIgnited.getFatBuild(id, syncMode))); + .map(b -> builds.computeIfAbsent(b.id(), id -> FutureUtil.getResult(loadBuildAsync(id, syncMode, teamcityIgnited)))); } if (includeLatestRebuild == LatestRebuildMode.ALL) { @@ -304,7 +311,7 @@ public class BuildChainProcessor { .sorted(Comparator.comparing(BuildRefCompacted::id).reversed()) .limit(cntLimit) // .filter(b -> !builds.containsKey(b.id())) // todo removing this causes incorrect count of failures (duplicated builds) - .map(b -> builds.computeIfAbsent(b.id(), id -> teamcityIgnited.getFatBuild(id, syncMode))); + .map(b -> builds.computeIfAbsent(b.id(), id -> FutureUtil.getResult(loadBuildAsync(id, syncMode, teamcityIgnited)))); } throw new UnsupportedOperationException("invalid mode " + includeLatestRebuild); @@ -365,47 +372,37 @@ public class BuildChainProcessor { return branch; } + /** + * @param buildFut Chain build future. + * @param mode Mode. + * @param builds Builds. + * @param teamcityIgnited Teamcity ignited. + * @return Set of new builds found during this dependencies check round. + */ @NotNull - private Stream<FatBuildCompacted> dependencies( - ITeamcityIgnited teamcityIgnited, - Map<Integer, FatBuildCompacted> builds, + private Set<Integer> dependencies( + Future<FatBuildCompacted> buildFut, SyncMode mode, - FatBuildCompacted build) { - - Stream<FatBuildCompacted> stream = IntStream.of(build.snapshotDependencies()) - .mapToObj(id -> { - if (builds.containsKey(id)) - return Futures.<FatBuildCompacted>immediateFuture(null); //load and propagate only new dependencies + Map<Integer, Future<FatBuildCompacted>> builds, + ITeamcityIgnited teamcityIgnited) { + Set<Integer> newBuilds = new HashSet<>(); - if (mode == SyncMode.NONE) - return Futures.immediateFuture(loadBuild(teamcityIgnited, builds, mode, id)); + IntStream.of(FutureUtil.getResult(buildFut).snapshotDependencies()) + .forEach(id -> builds.computeIfAbsent(id, id0 -> { + newBuilds.add(id0); - return tcUpdatePool.getService().submit(() -> { - if (builds.containsKey(id)) - return null; - - return loadBuild(teamcityIgnited, builds, mode, id); - }); - }) - .collect(Collectors.toList()) - .stream() - .map(future -> FutureUtil.getResult(future)) - .filter(Objects::nonNull); + return loadBuildAsync(id0, mode, teamcityIgnited); + })); - return Stream.concat( - Stream.of(build), - stream); + return newBuilds; } - @Nullable - private FatBuildCompacted loadBuild(ITeamcityIgnited teamcityIgnited, - Map<Integer, FatBuildCompacted> builds, - SyncMode mode, - int id) { - FatBuildCompacted buildLoaded = teamcityIgnited.getFatBuild(id, mode); + public Future<FatBuildCompacted> loadBuildAsync(Integer id, SyncMode mode, ITeamcityIgnited teamcityIgnited) { + if (mode == SyncMode.NONE) + return Futures.immediateFuture(teamcityIgnited.getFatBuild(id, SyncMode.NONE)); - FatBuildCompacted prevVal = builds.putIfAbsent(id, buildLoaded); - - return prevVal == null ? buildLoaded : null; + return tcUpdatePool.getService().submit(() -> { + return teamcityIgnited.getFatBuild(id, mode); + }); } } diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java index 2e2ec97..207f852 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java @@ -45,11 +45,11 @@ import org.apache.ignite.ci.teamcity.ignited.InMemoryStringCompactor; import org.apache.ignite.ci.teamcity.ignited.SyncMode; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; import org.jetbrains.annotations.NotNull; -import org.junit.Ignore; import org.junit.Test; import org.mockito.Mockito; import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertFalse; import static junit.framework.TestCase.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -60,8 +60,15 @@ import static org.mockito.Mockito.when; * Test for chain processor */ public class BuildChainProcessorTest { + /** Unique failed test, prefix for test name. This name will be unique each time. */ public static final String UNIQUE_FAILED_TEST = "uniqueFailedTest"; + + /** Test failing every time. */ public static final String TEST_FAILING_EVERY_TIME = "testFailingEveryTime"; + + /** Pds 1 build type ID. */ + public static final String PDS_1_BT_ID = "Pds1"; + /** Injector. */ private Injector injector = Guice.createInjector(new AbstractModule() { @Override protected void configure() { @@ -73,7 +80,6 @@ public class BuildChainProcessorTest { /** * */ - @Ignore @Test public void testAllBuildsArePresentInMergedBuilds() { IStringCompactor c = injector.getInstance(IStringCompactor.class); @@ -102,14 +108,42 @@ public class BuildChainProcessorTest { else assertTrue(suite.failedTests() >= 1); - List<ITestFailures> tests = suite.getFailedTests(); - for (ITestFailures test : tests) { + for (ITestFailures test : suite.getFailedTests()) { if (test.getName().startsWith(UNIQUE_FAILED_TEST)) assertEquals(1, test.failuresCount()); else if (test.getName().equals(TEST_FAILING_EVERY_TIME)) assertEquals(10, test.failuresCount()); } } + + //Adding successfull re-runs + for (int j = 0; j < 10; j++) { + FatBuildCompacted pds1 = testFatBuild(c, 130 + j, PDS_1_BT_ID); + pds1.buildTypeName(UNIQUE_FAILED_TEST, c); + + TestOccurrenceFull t1 = new TestOccurrenceFull(); + t1.name = UNIQUE_FAILED_TEST + j; + t1.status = TestOccurrence.STATUS_SUCCESS; + pds1.addTests(c, Lists.newArrayList(t1)); + + builds.put(pds1.id(), pds1); + } + + FullChainRunCtx ctx2 = bcp.loadFullChainContext(tcOldMock(), tcIgnited, + entry, + LatestRebuildMode.ALL, ProcessLogsMode.SUITE_NOT_COMPLETE, false, ITeamcity.DEFAULT, SyncMode.NONE); + List<MultBuildRunCtx> suites2 = ctx2.failedChildSuites().collect(Collectors.toList()); + + assertTrue(!suites2.isEmpty()); + + for (MultBuildRunCtx suite : suites) { + System.out.println(suite.getFailedTestsNames().collect(Collectors.toList())); + + if (suite.suiteName() != null && suite.suiteName().startsWith(UNIQUE_FAILED_TEST)) { + for (ITestFailures test : suite.getFailedTests()) + assertTrue("Failure found but should be hidden by re-run " + test, false); + } + } } /** @@ -144,7 +178,7 @@ public class BuildChainProcessorTest { builds.put(root.id(), root); - FatBuildCompacted pds1 = testFatBuild(c, 100 + i, "Pds1"); + FatBuildCompacted pds1 = testFatBuild(c, 100 + i, PDS_1_BT_ID); pds1.buildTypeName(UNIQUE_FAILED_TEST, c); TestOccurrenceFull t1 = new TestOccurrenceFull();