This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-9542-new-run-stripe
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/ignite-9542-new-run-stripe by
this push:
new 81ba33d IGNITE-9542 A number of optimization for initial loading
81ba33d is described below
commit 81ba33d0cd623263ed917cb9d288d1c7c9caf490
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Wed Nov 28 15:44:26 2018 +0300
IGNITE-9542 A number of optimization for initial loading
---
.../ci/teamcity/ignited/runhist/Invocation.java | 19 +++++++++
.../teamcity/ignited/runhist/InvocationData.java | 47 ++++++++++++++++++----
.../teamcity/ignited/runhist/RunHistCompacted.java | 20 ++++++++-
.../ignited/runhist/RunHistCompactedDao.java | 10 +++--
.../ci/teamcity/ignited/runhist/RunHistSync.java | 39 +++++++++---------
.../ci/web/model/current/SuiteCurrentStatus.java | 2 +-
6 files changed, 107 insertions(+), 30 deletions(-)
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/Invocation.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/Invocation.java
index 88e8510..f540c2f 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/Invocation.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/Invocation.java
@@ -18,6 +18,7 @@
package org.apache.ignite.ci.teamcity.ignited.runhist;
import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
public class Invocation {
@@ -64,4 +65,22 @@ public class Invocation {
public long startDate() {
return startDate;
}
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ Invocation that = (Invocation)o;
+ return buildId == that.buildId &&
+ status == that.status &&
+ changePresent == that.changePresent &&
+ startDate == that.startDate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hashCode(buildId, status, changePresent, startDate);
+ }
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java
index c740d15..a194c7c 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/InvocationData.java
@@ -18,6 +18,7 @@
package org.apache.ignite.ci.teamcity.ignited.runhist;
import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
import java.time.Duration;
import java.util.List;
import java.util.stream.Collectors;
@@ -55,30 +56,44 @@ public class InvocationData {
return allHistRuns;
}
- public boolean add(int build, Invocation inv) {
+ public boolean add(Invocation inv) {
+ try {
+ return innerAdd(inv);
+ }
+ finally {
+ removeEldiest();
+ }
+ }
+
+ public boolean innerAdd(Invocation inv) {
+ int build = inv.buildId();
+ if (build < 0)
+ return false;
+
if (invocationMap.containsKey(build))
return false;
+ if (isExpired(inv.startDate))
+ return false;
+
Invocation prevVal = invocationMap.putIfAbsent(build, inv);
- final boolean newValue = prevVal == null;
+ final boolean newVal = prevVal == null;
- if (newValue) {
+ if (newVal) {
allHistRuns++;
if (inv.isFailure())
allHistFailures++;
}
- removeEldiest();
-
- return newValue;
+ return newVal;
}
void removeEldiest() {
invocationMap.entrySet().removeIf(entries ->
isExpired(entries.getValue().startDate));
}
- public boolean isExpired(long startDate) {
+ public static boolean isExpired(long startDate) {
return (U.currentTimeMillis() - startDate) >
Duration.ofDays(MAX_DAYS).toMillis();
}
@@ -110,6 +125,7 @@ public class InvocationData {
.count();
}
+ /** {@inheritDoc} */
@Override public String toString() {
return MoreObjects.toStringHelper(this)
.add("allHistRuns", allHistRuns)
@@ -118,6 +134,23 @@ public class InvocationData {
.toString();
}
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ InvocationData data = (InvocationData)o;
+ return allHistRuns == data.allHistRuns &&
+ allHistFailures == data.allHistFailures &&
+ Objects.equal(invocationMap, data.invocationMap);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hashCode(allHistRuns, allHistFailures, invocationMap);
+ }
+
/**
*
*/
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
index f93d618..2841971 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompacted.java
@@ -18,6 +18,7 @@
package org.apache.ignite.ci.teamcity.ignited.runhist;
import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.ci.analysis.IVersionedEntity;
import org.apache.ignite.ci.db.Persisted;
@@ -96,9 +97,10 @@ public class RunHistCompacted implements IVersionedEntity,
IRunHistory {
/**
* @param build Build.
* @param inv Invocation.
+ * @return if test run is new and is not expired.
*/
public boolean addTestRun(int build, Invocation inv) {
- return data.add(build, inv);
+ return data.add(inv);
}
/** {@inheritDoc} */
@@ -110,4 +112,20 @@ public class RunHistCompacted implements IVersionedEntity,
IRunHistory {
.add("data", data)
.toString();
}
+
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ RunHistCompacted compacted = (RunHistCompacted)o;
+ return _ver == compacted._ver &&
+ testNameOrSuite == compacted.testNameOrSuite &&
+ srvId == compacted.srvId &&
+ Objects.equal(data, compacted.data);
+ }
+
+ @Override public int hashCode() {
+ return Objects.hashCode(_ver, testNameOrSuite, srvId, data);
+ }
}
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
index a4fa0c7..612249a 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistCompactedDao.java
@@ -100,28 +100,32 @@ public class RunHistCompactedDao {
@AutoProfiling
public Integer addInvocations(RunHistKey histKey, List<Invocation> list) {
- if(list.isEmpty())
+ if (list.isEmpty())
return 0;
return testHistCache.invoke(histKey, (entry, parms) -> {
int cnt = 0;
+
RunHistCompacted hist = entry.getValue();
if (hist == null)
hist = new RunHistCompacted(entry.getKey());
+ int initHashCode = hist.hashCode();
+
List<Invocation> invocationList = (List<Invocation>)parms[0];
for (Invocation invocation : invocationList) {
boolean added = hist.addTestRun(
invocation.buildId(),
invocation);
+
if (added)
cnt++;
-
}
- entry.setValue(hist);
+ if (cnt > 0 || hist.hashCode() != initHashCode)
+ entry.setValue(hist);
return cnt;
},
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
index d14a8ed..ade23df 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/runhist/RunHistSync.java
@@ -23,11 +23,9 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-import javax.annotation.Nullable;
import javax.annotation.concurrent.GuardedBy;
import javax.inject.Inject;
import org.apache.ignite.ci.di.AutoProfiling;
@@ -38,7 +36,6 @@ import org.apache.ignite.ci.teamcity.ignited.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
-import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,6 +46,8 @@ import org.slf4j.LoggerFactory;
public class RunHistSync {
/** Logger. */
private static final Logger logger =
LoggerFactory.getLogger(RunHistSync.class);
+ public static final int MAX_TESTS_QUEUE = 100000;
+ public static final int HIST_LDR_TASKS = 4;
/** Compactor. */
@Inject private IStringCompactor compactor;
@@ -97,7 +96,7 @@ public class RunHistSync {
synchronized (this) {
final SyncTask syncTask = buildToSave.computeIfAbsent(srvVame, s
-> new SyncTask());
- if (syncTask.sheduled() + cnt <= 100000)
+ if (syncTask.sheduled() + cnt <= MAX_TESTS_QUEUE)
syncTask.tests.putAll(data);
else
saveNow = true;
@@ -106,14 +105,16 @@ public class RunHistSync {
if(saveNow)
saveInvocationsMap(data);
else {
- scheduler.sheduleNamed(taskName("saveBuildToHistory", srvVame),
- () -> saveBuildToHistory(srvVame), 2, TimeUnit.MINUTES);
+ int ldrToActivate =
ThreadLocalRandom.current().nextInt(HIST_LDR_TASKS) + 1;
+
+ scheduler.sheduleNamed(taskName("saveBuildToHistory." +
ldrToActivate, srvVame),
+ () -> saveBuildToHistory(srvVame, ldrToActivate), 2,
TimeUnit.MINUTES);
}
}
- @MonitoredTask(name="Save Builds To History", nameExtArgIndex = 0)
+ @MonitoredTask(name="Save Builds To History(srv, runner)",
nameExtArgsIndexes = {0,1})
@SuppressWarnings("WeakerAccess")
- protected String saveBuildToHistory(String srvName) {
+ protected String saveBuildToHistory(String srvName, int ldrToActivate) {
Map<RunHistKey, List<Invocation>> saveThisRun;
synchronized (this) {
@@ -134,20 +135,20 @@ public class RunHistSync {
Set<Integer> confirmedNewBuild = new HashSet<>();
Set<Integer> confirmedDuplicate = new HashSet<>();
AtomicInteger invocations = new AtomicInteger();
- AtomicInteger duplicates = new AtomicInteger();
+ AtomicInteger duplicateOrExpired = new AtomicInteger();
saveThisRun.forEach(
(histKey, invocationList) -> {
saveInvocationList(confirmedNewBuild,
confirmedDuplicate,
invocations,
- duplicates,
+ duplicateOrExpired,
histKey, invocationList);
}
);
String res = "History entries: " + saveThisRun.size() + " processed "
+ invocations.get()
- + " invocations saved to DB " + duplicates.get() + " duplicates";
+ + " invocations saved to DB " + duplicateOrExpired.get() + "
duplicates/expired";
System.out.println(Thread.currentThread().getName() + ":" + res);
@@ -157,16 +158,17 @@ public class RunHistSync {
private void saveInvocationList(Set<Integer> confirmedNewBuild,
Set<Integer> confirmedDuplicate,
AtomicInteger invocations,
- AtomicInteger duplicates,
+ AtomicInteger duplicateOrExpired,
RunHistKey histKey,
List<Invocation> invocationList) {
- List<Invocation> newInvocations = new ArrayList<>();
+ List<Invocation> invocationsToSave = new ArrayList<>();
invocationList.forEach(
inv -> {
int buildId = inv.buildId();
if (confirmedNewBuild.contains(buildId)) {
- newInvocations.add(inv);
+ if (!InvocationData.isExpired(inv.startDate()))
+ invocationsToSave.add(inv);
return;
}
@@ -177,17 +179,18 @@ public class RunHistSync {
if (histDao.setBuildProcessed(histKey.srvId(), buildId,
inv.startDate())) {
confirmedNewBuild.add(buildId);
- newInvocations.add(inv);
+ if (!InvocationData.isExpired(inv.startDate()))
+ invocationsToSave.add(inv);
}
else
confirmedDuplicate.add(buildId);
}
);
- Integer cntAdded = histDao.addInvocations(histKey, newInvocations);
+ Integer cntAdded = histDao.addInvocations(histKey, invocationsToSave);
invocations.addAndGet(cntAdded);
- duplicates.addAndGet(invocationList.size() - cntAdded);
+ duplicateOrExpired.addAndGet(invocationList.size() - cntAdded);
}
public void invokeLaterFindMissingHistory(String srvName) {
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
index ab724fc..3348e8d 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/model/current/SuiteCurrentStatus.java
@@ -52,7 +52,7 @@ import static org.apache.ignite.ci.util.UrlUtil.escape;
* Represent Suite result
*/
@SuppressWarnings("WeakerAccess") public class SuiteCurrentStatus extends
FailureSummary {
- public static final boolean NEW_RUN_STAT = false;
+ public static final boolean NEW_RUN_STAT = true;
/** Suite Name */
public String name;
