This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-10336
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 5d3aad65544903f6d77afc222bf8160cce8afd2a
Author: Dmitriy Pavlov <dpav...@apache.org>
AuthorDate: Wed Nov 21 15:33:06 2018 +0300
IGNITE-10336 Chain collection refactoring started
---
.../ci/tcbot/builds/CompareBuildsService.java | 2 +-
.../ignite/ci/tcbot/chain/BuildChainProcessor.java | 131 ++++++++++-----------
.../ci/tcbot/chain/BuildChainProcessorTest.java | 44 ++++++-
3 files changed, 104 insertions(+), 73 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
index 71715c8..e420ca4 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/builds/CompareBuildsService.java
@@ -80,7 +80,7 @@ public class CompareBuildsService {
MultBuildRunCtx buildCtx = new MultBuildRunCtx(build, compactor);
final FatBuildCompacted fatBuild =
tcIgnited.getFatBuild(build.getId());
- buildCtx.addBuild(bcp.loadTestsAndProblems(fatBuild, tcIgnited));
+ buildCtx.addBuild(bcp.loadChanges(fatBuild, tcIgnited));
for (String testName : buildCtx.tests())
tests.add(extractTestName(testName));
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
index 14d6d2e..66cfef8 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessor.java
@@ -17,14 +17,17 @@
package org.apache.ignite.ci.tcbot.chain;
+import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.Futures;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -101,12 +104,15 @@ public class BuildChainProcessor {
final ExecutorService svc = tcUpdatePool.getService();
- final Stream<FatBuildCompacted> depsFirstLevel = entryPointsFatBuilds
+ final Stream<FatBuildCompacted> depsFirstLevel =
+ Stream.of(); //todo implement
+ /*entryPointsFatBuilds
.map(ref -> svc.submit(() -> dependencies(teamcityIgnited, builds,
mode, ref)))
- .collect(Collectors.toList())
+ .flatMap(set->set.stream())
.stream()
.flatMap(fut -> FutureUtil.getResult(fut));
+*/
depsFirstLevel
.filter(b -> !b.isComposite() && b.getTestsCount() > 0)
.forEach(b ->
@@ -121,7 +127,7 @@ public class BuildChainProcessor {
);
if (!lrTests.isEmpty()) {
- Collections.sort(lrTests, (test0, test1) -> {
+ lrTests.sort((test0, test1) -> {
long t0 = test0.time;
long t1 = test1.time;
@@ -156,18 +162,18 @@ public class BuildChainProcessor {
/**
* @param teamcity Teamcity.
- * @param teamcityIgnited
+ * @param tcIgn Teamcity Ignited.
* @param entryPoints Entry point(s): Build(s) to start scan from.
* @param includeLatestRebuild Include latest rebuild.
* @param procLog Process logger.
* @param includeScheduledInfo Include scheduled info.
* @param failRateBranch Fail rate branch.
- * @param mode
+ * @param mode background data update mode.
*/
@AutoProfiling
public FullChainRunCtx loadFullChainContext(
IAnalyticsEnabledTeamcity teamcity,
- ITeamcityIgnited teamcityIgnited,
+ ITeamcityIgnited tcIgn,
Collection<Integer> entryPoints,
LatestRebuildMode includeLatestRebuild,
ProcessLogsMode procLog,
@@ -178,39 +184,41 @@ public class BuildChainProcessor {
if (entryPoints.isEmpty())
return new FullChainRunCtx(Build.createFakeStub());
- Map<Integer, FatBuildCompacted> builds = new ConcurrentHashMap<>();
+ Map<Integer, Future<FatBuildCompacted>> builds = new
ConcurrentHashMap<>();
- final Stream<FatBuildCompacted> entryPointsFatBuilds =
entryPoints.stream()
- .filter(Objects::nonNull)
- .filter(id -> !builds.containsKey(id)) //load and propagate
only new entry points
- .map(id -> builds.computeIfAbsent(id, id0 ->
teamcityIgnited.getFatBuild(id0, mode)));
+ Stream<Future<FatBuildCompacted>> entryPointsFatBuilds =
entryPoints.stream()
+ .filter(Objects::nonNull)
+ .map(id -> builds.computeIfAbsent(id, id0 -> loadBuildAsync(id0,
mode, tcIgn)));
- final ExecutorService svc = tcUpdatePool.getService();
+ Stream<Integer> dependenciesFirstLevel = entryPointsFatBuilds
+ .flatMap(ref -> dependencies(ref, mode, builds, tcIgn).stream());
- final Stream<FatBuildCompacted> depsFirstLevel = entryPointsFatBuilds
- .flatMap(ref -> dependencies(teamcityIgnited, builds, mode, ref));
+ Stream<Integer> depsSecondLevel =
dependenciesFirstLevel.map(builds::get)
+ .peek(val -> Preconditions.checkNotNull(val, "Build future should
be in context"))
+ .flatMap(ref -> dependencies(ref, mode, builds, tcIgn).stream());
- Stream<FatBuildCompacted> secondLevelDeps = depsFirstLevel
- .flatMap(ref -> dependencies(teamcityIgnited, builds, mode, ref));
+ Set<Integer> collect = depsSecondLevel.collect(Collectors.toSet());
+ System.err.println("New deps of second level:" + collect);
// builds may became non unique because of race in filtering and
acquiring deps
- final List<Future<Stream<FatBuildCompacted>>> phase3Submitted =
secondLevelDeps
+ //todo implement
+ /* final List<Future<Stream<FatBuildCompacted>>> phase3Submitted =
secondLevelDeps
.map((fatBuild) -> svc.submit(
- () -> replaceWithRecent(teamcityIgnited,
includeLatestRebuild, mode, builds, fatBuild, entryPoints.size())))
- .collect(Collectors.toList());
+ () -> replaceWithRecent(tcIgn, includeLatestRebuild,
mode, builds, fatBuild, entryPoints.size())))
+ .collect(Collectors.toList());*/
- Map<String, MultBuildRunCtx> buildsCtxMap = new ConcurrentHashMap<>();
+ Map<String, MultBuildRunCtx> buildsByBt = new ConcurrentHashMap<>();
- phase3Submitted.stream()
- .flatMap(fut -> FutureUtil.getResult(fut))
- .forEach((fatBuild) -> createCxt(teamcityIgnited,
buildsCtxMap, fatBuild));
+ builds.values().stream()
+ .map(FutureUtil::getResult)
+ .forEach((fatBuild) -> createCxt(tcIgn, buildsByBt, fatBuild));
- ArrayList<MultBuildRunCtx> contexts = new
ArrayList<>(buildsCtxMap.values());
+ ArrayList<MultBuildRunCtx> contexts = new
ArrayList<>(buildsByBt.values());
contexts.forEach(multiCtx -> {
analyzeTests(multiCtx, teamcity, procLog);
- fillBuildCounts(multiCtx, teamcityIgnited, includeScheduledInfo);
+ fillBuildCounts(multiCtx, tcIgn, includeScheduledInfo);
});
Function<MultBuildRunCtx, Float> function = ctx -> {
@@ -227,8 +235,8 @@ public class BuildChainProcessor {
};
Integer someEntryPnt = entryPoints.iterator().next();
- FatBuildCompacted build = builds.computeIfAbsent(someEntryPnt, id ->
teamcityIgnited.getFatBuild(id, mode));
- FullChainRunCtx fullChainRunCtx = new
FullChainRunCtx(build.toBuild(compactor));
+ Future<FatBuildCompacted> build = builds.computeIfAbsent(someEntryPnt,
id -> loadBuildAsync(id, mode, tcIgn));
+ FullChainRunCtx fullChainRunCtx = new
FullChainRunCtx(FutureUtil.getResult(build).toBuild(compactor));
contexts.sort(Comparator.comparing(function).reversed());
@@ -237,6 +245,7 @@ public class BuildChainProcessor {
return fullChainRunCtx;
}
+
@SuppressWarnings("WeakerAccess")
@AutoProfiling
protected void createCxt(ITeamcityIgnited teamcityIgnited,
@@ -250,7 +259,7 @@ public class BuildChainProcessor {
final MultBuildRunCtx ctx =
buildsCtxMap.computeIfAbsent(ref.buildTypeId,
k -> new MultBuildRunCtx(ref, compactor));
- ctx.addBuild(loadTestsAndProblems(buildCompacted, teamcityIgnited));
+ ctx.addBuild(loadChanges(buildCompacted, teamcityIgnited));
}
/**
@@ -260,14 +269,12 @@ public class BuildChainProcessor {
* @param tcIgnited
* @return Full context.
*/
- public SingleBuildRunCtx loadTestsAndProblems(@Nonnull FatBuildCompacted
buildCompacted,
+ public SingleBuildRunCtx loadChanges(@Nonnull FatBuildCompacted
buildCompacted,
ITeamcityIgnited tcIgnited) {
SingleBuildRunCtx ctx = new SingleBuildRunCtx(buildCompacted,
compactor);
ctx.setChanges(tcIgnited.getAllChanges(buildCompacted.changes()));
- //todo support storing build.lastChanges.changes) ?
-
return ctx;
}
@@ -296,7 +303,7 @@ public class BuildChainProcessor {
.orElse(buildCompacted);
return Stream.of(recentRef)
- .map(b -> builds.computeIfAbsent(b.id(), id ->
teamcityIgnited.getFatBuild(id, syncMode)));
+ .map(b -> builds.computeIfAbsent(b.id(), id ->
FutureUtil.getResult(loadBuildAsync(id, syncMode, teamcityIgnited))));
}
if (includeLatestRebuild == LatestRebuildMode.ALL) {
@@ -304,7 +311,7 @@ public class BuildChainProcessor {
.sorted(Comparator.comparing(BuildRefCompacted::id).reversed())
.limit(cntLimit)
// .filter(b -> !builds.containsKey(b.id())) // todo removing
this causes incorrect count of failures (duplicated builds)
- .map(b -> builds.computeIfAbsent(b.id(), id ->
teamcityIgnited.getFatBuild(id, syncMode)));
+ .map(b -> builds.computeIfAbsent(b.id(), id ->
FutureUtil.getResult(loadBuildAsync(id, syncMode, teamcityIgnited))));
}
throw new UnsupportedOperationException("invalid mode " +
includeLatestRebuild);
@@ -365,47 +372,37 @@ public class BuildChainProcessor {
return branch;
}
+ /**
+ * @param buildFut Chain build future.
+ * @param mode Mode.
+ * @param builds Builds.
+ * @param teamcityIgnited Teamcity ignited.
+ * @return Set of new builds found during this dependencies check round.
+ */
@NotNull
- private Stream<FatBuildCompacted> dependencies(
- ITeamcityIgnited teamcityIgnited,
- Map<Integer, FatBuildCompacted> builds,
+ private Set<Integer> dependencies(
+ Future<FatBuildCompacted> buildFut,
SyncMode mode,
- FatBuildCompacted build) {
-
- Stream<FatBuildCompacted> stream =
IntStream.of(build.snapshotDependencies())
- .mapToObj(id -> {
- if (builds.containsKey(id))
- return Futures.<FatBuildCompacted>immediateFuture(null);
//load and propagate only new dependencies
+ Map<Integer, Future<FatBuildCompacted>> builds,
+ ITeamcityIgnited teamcityIgnited) {
+ Set<Integer> newBuilds = new HashSet<>();
- if (mode == SyncMode.NONE)
- return Futures.immediateFuture(loadBuild(teamcityIgnited,
builds, mode, id));
+ IntStream.of(FutureUtil.getResult(buildFut).snapshotDependencies())
+ .forEach(id -> builds.computeIfAbsent(id, id0 -> {
+ newBuilds.add(id0);
- return tcUpdatePool.getService().submit(() -> {
- if (builds.containsKey(id))
- return null;
-
- return loadBuild(teamcityIgnited, builds, mode, id);
- });
- })
- .collect(Collectors.toList())
- .stream()
- .map(future -> FutureUtil.getResult(future))
- .filter(Objects::nonNull);
+ return loadBuildAsync(id0, mode, teamcityIgnited);
+ }));
- return Stream.concat(
- Stream.of(build),
- stream);
+ return newBuilds;
}
- @Nullable
- private FatBuildCompacted loadBuild(ITeamcityIgnited teamcityIgnited,
- Map<Integer, FatBuildCompacted> builds,
- SyncMode mode,
- int id) {
- FatBuildCompacted buildLoaded = teamcityIgnited.getFatBuild(id, mode);
+ public Future<FatBuildCompacted> loadBuildAsync(Integer id, SyncMode mode,
ITeamcityIgnited teamcityIgnited) {
+ if (mode == SyncMode.NONE)
+ return Futures.immediateFuture(teamcityIgnited.getFatBuild(id,
SyncMode.NONE));
- FatBuildCompacted prevVal = builds.putIfAbsent(id, buildLoaded);
-
- return prevVal == null ? buildLoaded : null;
+ return tcUpdatePool.getService().submit(() -> {
+ return teamcityIgnited.getFatBuild(id, mode);
+ });
}
}
diff --git
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java
index 2e2ec97..207f852 100644
---
a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java
+++
b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/BuildChainProcessorTest.java
@@ -45,11 +45,11 @@ import
org.apache.ignite.ci.teamcity.ignited.InMemoryStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.SyncMode;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.jetbrains.annotations.NotNull;
-import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertFalse;
import static junit.framework.TestCase.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
@@ -60,8 +60,15 @@ import static org.mockito.Mockito.when;
* Test for chain processor
*/
public class BuildChainProcessorTest {
+ /** Unique failed test, prefix for test name. This name will be unique
each time. */
public static final String UNIQUE_FAILED_TEST = "uniqueFailedTest";
+
+ /** Test failing every time. */
public static final String TEST_FAILING_EVERY_TIME =
"testFailingEveryTime";
+
+ /** Pds 1 build type ID. */
+ public static final String PDS_1_BT_ID = "Pds1";
+
/** Injector. */
private Injector injector = Guice.createInjector(new AbstractModule() {
@Override protected void configure() {
@@ -73,7 +80,6 @@ public class BuildChainProcessorTest {
/**
*
*/
- @Ignore
@Test
public void testAllBuildsArePresentInMergedBuilds() {
IStringCompactor c = injector.getInstance(IStringCompactor.class);
@@ -102,14 +108,42 @@ public class BuildChainProcessorTest {
else
assertTrue(suite.failedTests() >= 1);
- List<ITestFailures> tests = suite.getFailedTests();
- for (ITestFailures test : tests) {
+ for (ITestFailures test : suite.getFailedTests()) {
if (test.getName().startsWith(UNIQUE_FAILED_TEST))
assertEquals(1, test.failuresCount());
else if (test.getName().equals(TEST_FAILING_EVERY_TIME))
assertEquals(10, test.failuresCount());
}
}
+
+ //Adding successfull re-runs
+ for (int j = 0; j < 10; j++) {
+ FatBuildCompacted pds1 = testFatBuild(c, 130 + j, PDS_1_BT_ID);
+ pds1.buildTypeName(UNIQUE_FAILED_TEST, c);
+
+ TestOccurrenceFull t1 = new TestOccurrenceFull();
+ t1.name = UNIQUE_FAILED_TEST + j;
+ t1.status = TestOccurrence.STATUS_SUCCESS;
+ pds1.addTests(c, Lists.newArrayList(t1));
+
+ builds.put(pds1.id(), pds1);
+ }
+
+ FullChainRunCtx ctx2 = bcp.loadFullChainContext(tcOldMock(), tcIgnited,
+ entry,
+ LatestRebuildMode.ALL, ProcessLogsMode.SUITE_NOT_COMPLETE, false,
ITeamcity.DEFAULT, SyncMode.NONE);
+ List<MultBuildRunCtx> suites2 =
ctx2.failedChildSuites().collect(Collectors.toList());
+
+ assertTrue(!suites2.isEmpty());
+
+ for (MultBuildRunCtx suite : suites) {
+
System.out.println(suite.getFailedTestsNames().collect(Collectors.toList()));
+
+ if (suite.suiteName() != null &&
suite.suiteName().startsWith(UNIQUE_FAILED_TEST)) {
+ for (ITestFailures test : suite.getFailedTests())
+ assertTrue("Failure found but should be hidden by re-run "
+ test, false);
+ }
+ }
}
/**
@@ -144,7 +178,7 @@ public class BuildChainProcessorTest {
builds.put(root.id(), root);
- FatBuildCompacted pds1 = testFatBuild(c, 100 + i, "Pds1");
+ FatBuildCompacted pds1 = testFatBuild(c, 100 + i, PDS_1_BT_ID);
pds1.buildTypeName(UNIQUE_FAILED_TEST, c);
TestOccurrenceFull t1 = new TestOccurrenceFull();
