This is an automated email from the ASF dual-hosted git repository.
dspavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new 95c63c2a IGNITE-28630: Improve TC Bot diagnostics and add legacy DB
compatibility test (#211)
95c63c2a is described below
commit 95c63c2a5bb56092f19f59b86f1847135aaa16e4
Author: ignitetcbot <[email protected]>
AuthorDate: Fri May 8 20:22:07 2026 +0300
IGNITE-28630: Improve TC Bot diagnostics and add legacy DB compatibility
test (#211)
Add manual legacy DB compatibility perf test excluded from regular CI.
---
.gitignore | 2 +-
README.md | 3 +
migrator/build.gradle | 20 +++++
.../LegacyPersistentStorageCompatibilityTest.java | 87 +++++++++++++++++++++-
tcbot-engine/build.gradle | 2 +-
tcbot-github-ignited/build.gradle | 2 +-
tcbot-jira-ignited/build.gradle | 2 +-
tcbot-jira/build.gradle | 4 +-
tcbot-persistence/build.gradle | 2 +-
tcbot-teamcity-ignited/build.gradle | 4 +-
.../ignite/tcignited/buildref/BuildRefDao.java | 59 +++++++++++----
tcbot-teamcity/build.gradle | 4 +-
12 files changed, 163 insertions(+), 28 deletions(-)
diff --git a/.gitignore b/.gitignore
index dc3fbd97..b8938992 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,4 +35,4 @@ hs_err_pid*
/conf/branches-prom.json
/ignite-tc-helper-web/ignite/**
/ignite-tc-helper-web/src/test/tmp/**
-/migrator/src/test/work/**
\ No newline at end of file
+/migrator/src/test/work/**
diff --git a/README.md b/README.md
index 5ca5ed51..1f7a2f69 100644
--- a/README.md
+++ b/README.md
@@ -125,3 +125,6 @@ be retried instead of silently leaving mixed old and new
data.
The same migrator can also be run as a standalone tool from the `migrator`
module against an Ignite work directory. The
standalone module uses the same Ignite version as the rest of the project
through the shared `ignVer` Gradle property.
+The heavyweight legacy storage compatibility/perf test is excluded from the
regular `:migrator:test` task. Run it
+explicitly with `./gradlew :migrator:legacyDbCompatPerfTest --no-daemon` when
checking old Ignite 2.14 persistent
+storage compatibility.
diff --git a/migrator/build.gradle b/migrator/build.gradle
index d47fd114..4e6a9140 100644
--- a/migrator/build.gradle
+++ b/migrator/build.gradle
@@ -26,6 +26,26 @@ test {
jvmArgs igniteJava17JvmArgs
maxHeapSize = '1536m'
systemProperty 'compat.work.dir',
file('src/test/work/ignite-db-compat').absolutePath
+ exclude '**/LegacyPersistentStorageCompatibilityTest.class'
+ failOnNoDiscoveredTests = false
+
+ testLogging {
+ events 'passed', 'failed', 'skipped', 'standardOut', 'standardError'
+ showStandardStreams = true
+ }
+}
+
+tasks.register('legacyDbCompatPerfTest', Test) {
+ description = 'Runs the heavyweight legacy Ignite persistent storage
compatibility/perf test.'
+ group = 'verification'
+
+ testClassesDirs = sourceSets.test.output.classesDirs
+ classpath = sourceSets.test.runtimeClasspath
+
+ jvmArgs igniteJava17JvmArgs
+ maxHeapSize = '1536m'
+ systemProperty 'compat.work.dir',
file('src/test/work/ignite-db-compat').absolutePath
+ include '**/LegacyPersistentStorageCompatibilityTest.class'
testLogging {
events 'passed', 'failed', 'skipped', 'standardOut', 'standardError'
diff --git
a/migrator/src/test/java/org/apache/ignite/migrate/LegacyPersistentStorageCompatibilityTest.java
b/migrator/src/test/java/org/apache/ignite/migrate/LegacyPersistentStorageCompatibilityTest.java
index 6dcd5d17..7c5cfb58 100644
---
a/migrator/src/test/java/org/apache/ignite/migrate/LegacyPersistentStorageCompatibilityTest.java
+++
b/migrator/src/test/java/org/apache/ignite/migrate/LegacyPersistentStorageCompatibilityTest.java
@@ -105,6 +105,9 @@ public class LegacyPersistentStorageCompatibilityTest {
/** Cache used only to produce enough WAL with old Ignite. */
private static final String LEGACY_WAL_STRESS_CACHE =
"legacyWalStressCache";
+ /** Current migration key that must be absent in the legacy marker set and
then applied by current code. */
+ private static final String GRID_INT_LIST_MIGRATION =
"migrate-GridIntList";
+
/** Small durable region for the test. */
private static final long REGION_SIZE = 256L * 1024 * 1024;
@@ -144,10 +147,15 @@ public class LegacyPersistentStorageCompatibilityTest {
ignite.cluster().active(true);
+ IgniteCache<String, Object> doneMigrations =
assertLegacyMigrationMarkersExist(ignite);
+
String migrationRes = runMigrationsWithFilteredOutput(ignite);
System.out.println("Migration result: " + migrationRes);
+ assertTrue("Current migrator must record the GridIntList
migration",
+ doneMigrations.containsKey(GRID_INT_LIST_MIGRATION));
+
PersistentStringCompactor compactor = new
PersistentStringCompactor(ignite);
int srvId = ITeamcityIgnited.serverIdToInt(SRV_ID);
@@ -172,6 +180,8 @@ public class LegacyPersistentStorageCompatibilityTest {
assertEquals(GridIntList.asList(1, 2, 3), migrated);
+ assertAllUserCachesCanBeDeserialized(ignite);
+
failureHandler.assertNoFailure();
}
finally {
@@ -225,7 +235,7 @@ public class LegacyPersistentStorageCompatibilityTest {
if (line.startsWith("cache [") && line.endsWith("] not found"))
missingCaches++;
- if (line.contains("migrate-GridIntList"))
+ if (line.contains(GRID_INT_LIST_MIGRATION))
gridIntListLines.add(line);
}
@@ -237,6 +247,74 @@ public class LegacyPersistentStorageCompatibilityTest {
assertTrue("GridIntList migration must run",
!gridIntListLines.isEmpty());
}
+ /**
+ * @param ignite Ignite.
+ * @return Existing done migrations cache created by the old bot code.
+ */
+ private IgniteCache<String, Object>
assertLegacyMigrationMarkersExist(Ignite ignite) {
+ String cacheName =
DbMigrations.ignCacheNme(DbMigrations.DONE_MIGRATIONS,
DbMigrations.DONE_MIGRATION_PREFIX);
+ IgniteCache<String, Object> doneMigrations = ignite.cache(cacheName);
+
+ assertNotNull("Legacy generator must create old migration markers
cache " + cacheName, doneMigrations);
+
+ int size = doneMigrations.size();
+
+ System.out.println("Legacy migration markers before current migrator:
" + size);
+
+ assertTrue("Legacy generator must execute old migrations and leave
done markers", size > 0);
+ assertFalse("Legacy marker set must not already skip the current
GridIntList migration",
+ doneMigrations.containsKey(GRID_INT_LIST_MIGRATION));
+
+ return doneMigrations;
+ }
+
+ /**
+ * @param ignite Ignite.
+ */
+ private void assertAllUserCachesCanBeDeserialized(Ignite ignite) {
+ int caches = 0;
+ long entries = 0;
+
+ for (String cacheName : ignite.cacheNames()) {
+ IgniteCache<Object, Object> cache = ignite.cache(cacheName);
+
+ if (cache == null)
+ continue;
+
+ caches++;
+
+ long cacheEntries = 0;
+
+ try {
+ for (Cache.Entry<Object, Object> entry : cache) {
+ Object key = entry.getKey();
+ Object val = entry.getValue();
+
+ if (key != null)
+ key.getClass();
+
+ if (val != null)
+ val.getClass();
+
+ cacheEntries++;
+ }
+ }
+ catch (RuntimeException | LinkageError e) {
+ throw new AssertionError("Unable to deserialize cache [" +
cacheName
+ + "] after migration, processedEntries=" + cacheEntries,
e);
+ }
+
+ entries += cacheEntries;
+
+ System.out.println("Deserialized cache [" + cacheName + "]
entries=" + cacheEntries);
+ }
+
+ System.out.println("Deserialized persistent caches: caches=" + caches
+ ", entries=" + entries);
+
+ assertTrue("At least one cache must be checked", caches > 0);
+ assertTrue("At least one cache entry must be checked", entries > 0);
+ }
+
/**
* @param line Migration log line.
* @return Migration procedure name.
@@ -539,6 +617,11 @@ public class LegacyPersistentStorageCompatibilityTest {
+ "\n"
+ " writeWalStressData(ignite);\n"
+ "\n"
+ + " String legacyMigrationRes = new
DbMigrations(ignite).dataMigration();\n"
+ + " IgniteCache<String, Object> doneMigrations =
ignite.cache(DbMigrations.ignCacheNme(DbMigrations.DONE_MIGRATIONS,
DbMigrations.DONE_MIGRATION_PREFIX));\n"
+ + " System.out.println(\"Legacy migrations result: \" +
legacyMigrationRes);\n"
+ + " System.out.println(\"Legacy migration markers: \" +
doneMigrations.size());\n"
+ + "\n"
+ " System.out.println(\"Legacy DB generated at: \" +
workDir.getAbsolutePath());\n"
+ " }\n"
+ " }\n"
@@ -889,6 +972,8 @@ public class LegacyPersistentStorageCompatibilityTest {
return line.startsWith("> Task")
|| line.startsWith("BUILD ")
|| line.contains("Legacy generator")
+ || line.contains("Legacy migrations")
+ || line.contains("Legacy migration markers")
|| line.contains("Legacy WAL stress")
|| line.contains("Legacy Ignite node version")
|| line.contains("Legacy DB generated")
diff --git a/tcbot-engine/build.gradle b/tcbot-engine/build.gradle
index f557ba6c..3606f363 100644
--- a/tcbot-engine/build.gradle
+++ b/tcbot-engine/build.gradle
@@ -26,4 +26,4 @@ dependencies {
testImplementation group: 'junit', name: 'junit', version: junitVer
testImplementation group: 'org.mockito', name: 'mockito-core', version:
mockitoVer
}
-
+
diff --git a/tcbot-github-ignited/build.gradle
b/tcbot-github-ignited/build.gradle
index cdd1ff42..bc131e9b 100644
--- a/tcbot-github-ignited/build.gradle
+++ b/tcbot-github-ignited/build.gradle
@@ -21,4 +21,4 @@ dependencies {
api project(":tcbot-github")
api project(":tcbot-persistence")
}
-
+
diff --git a/tcbot-jira-ignited/build.gradle b/tcbot-jira-ignited/build.gradle
index 5070b4e7..bb41aa53 100644
--- a/tcbot-jira-ignited/build.gradle
+++ b/tcbot-jira-ignited/build.gradle
@@ -21,4 +21,4 @@ dependencies {
api project(":tcbot-jira")
api project(":tcbot-persistence")
}
-
+
diff --git a/tcbot-jira/build.gradle b/tcbot-jira/build.gradle
index ceaedf78..ffafe364 100644
--- a/tcbot-jira/build.gradle
+++ b/tcbot-jira/build.gradle
@@ -21,8 +21,8 @@ dependencies {
api project(":tcbot-common")
// JIRA integration shares entries for pure and Ignited connection, so
persistence module is here for interfaces/annotations
api project(":tcbot-persistence")
-
+
testImplementation group: 'junit', name: 'junit', version: junitVer
testImplementation group: 'org.mockito', name: 'mockito-core', version:
mockitoVer
}
-
+
diff --git a/tcbot-persistence/build.gradle b/tcbot-persistence/build.gradle
index e619aef1..baa488d3 100644
--- a/tcbot-persistence/build.gradle
+++ b/tcbot-persistence/build.gradle
@@ -31,4 +31,4 @@ dependencies {
api "org.apache.ignite:ignite-indexing:$ignVer"
}
-
+
diff --git a/tcbot-teamcity-ignited/build.gradle
b/tcbot-teamcity-ignited/build.gradle
index 1720ef52..6135679d 100644
--- a/tcbot-teamcity-ignited/build.gradle
+++ b/tcbot-teamcity-ignited/build.gradle
@@ -16,7 +16,7 @@
*/
apply plugin: 'java'
-
+
dependencies {
api project(":tcbot-teamcity")
api project(":tcbot-persistence")
@@ -24,4 +24,4 @@ dependencies {
testImplementation group: 'junit', name: 'junit', version: junitVer;
testImplementation group: 'org.mockito', name: 'mockito-core', version:
mockitoVer;
}
-
+
diff --git
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
index 771678e6..d4aef425 100644
---
a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
+++
b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java
@@ -28,6 +28,7 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -74,6 +75,9 @@ public class BuildRefDao {
/** Update Counters for branch-related changes storage. */
@Inject private UpdateCountersStorage countersStorage;
+ /** Guard for branch history in-memory caches. */
+ private final ReentrantReadWriteLock buildRefsInMemCacheLock = new
ReentrantReadWriteLock(true);
+
/** Non persistence cache for all BuildRefsCompacted for particular branch.
* RunHistKey(ServerId||BranchId||suiteId)-> Build reference
*/
@@ -182,8 +186,15 @@ public class BuildRefDao {
.map(b -> branchNameToHistCacheKey(srvId, b.branch()))
.collect(Collectors.toSet());
- buildRefsInMemCacheForAllBranch.invalidateAll(cacheForAllBranch);
- buildRefsInMemCache.invalidateAll(setOfHistToClear);
+ buildRefsInMemCacheLock.writeLock().lock();
+
+ try {
+ buildRefsInMemCacheForAllBranch.invalidateAll(cacheForAllBranch);
+ buildRefsInMemCache.invalidateAll(setOfHistToClear);
+ }
+ finally {
+ buildRefsInMemCacheLock.writeLock().unlock();
+ }
setOfHistToClear.forEach(b -> {
int branch = b.branch();
@@ -228,22 +239,30 @@ public class BuildRefDao {
branchNameIds.forEach(branchNameId -> {
RunHistKey runHistKey = new RunHistKey(srvId, buildTypeIdId,
branchNameId);
try {
- List<BuildRefCompacted> compactedBuildsForBranch =
- buildRefsInMemCache.get(runHistKey, () -> {
- List<BuildRefCompacted> branch =
getBuildsForBranch(srvId, branchNameId);
+ List<BuildRefCompacted> compactedBuildsForBranch;
+
+ buildRefsInMemCacheLock.readLock().lock();
- List<BuildRefCompacted> resForBranch = branch.stream()
- .filter(e -> e.buildTypeId() == buildTypeIdId)
- .collect(Collectors.toList());
+ try {
+ compactedBuildsForBranch =
buildRefsInMemCache.get(runHistKey, () -> {
+ List<BuildRefCompacted> branch =
getBuildsForBranch(srvId, branchNameId);
- if (!resForBranch.isEmpty()) {
- System.err.println("Branch " +
compactor.getStringFromId(branchNameId)
- + " Suite " +
compactor.getStringFromId(buildTypeIdId)
- + " builds " + resForBranch.size() + " ");
- }
+ List<BuildRefCompacted> resForBranch =
branch.stream()
+ .filter(e -> e.buildTypeId() == buildTypeIdId)
+ .collect(Collectors.toList());
- return resForBranch;
- });
+ if (!resForBranch.isEmpty()) {
+ System.err.println("Branch " +
compactor.getStringFromId(branchNameId)
+ + " Suite " +
compactor.getStringFromId(buildTypeIdId)
+ + " builds " + resForBranch.size() + " ");
+ }
+
+ return resForBranch;
+ });
+ }
+ finally {
+ buildRefsInMemCacheLock.readLock().unlock();
+ }
res.addAll(compactedBuildsForBranch);
}
@@ -289,7 +308,15 @@ public class BuildRefDao {
long branchKey = branchNameToHistCacheKey(srvId, branchNameId);
try {
- return buildRefsInMemCacheForAllBranch.get(branchKey, () ->
getBuildsForBranchNonCached(srvId, branchNameId));
+ buildRefsInMemCacheLock.readLock().lock();
+
+ try {
+ return buildRefsInMemCacheForAllBranch.get(branchKey,
+ () -> getBuildsForBranchNonCached(srvId, branchNameId));
+ }
+ finally {
+ buildRefsInMemCacheLock.readLock().unlock();
+ }
}
catch (ExecutionException e) {
throw ExceptionUtil.propagateException(e);
diff --git a/tcbot-teamcity/build.gradle b/tcbot-teamcity/build.gradle
index 7f18daa5..53211fe1 100644
--- a/tcbot-teamcity/build.gradle
+++ b/tcbot-teamcity/build.gradle
@@ -16,7 +16,7 @@
*/
apply plugin: 'java'
-
+
dependencies {
api project(":tcbot-common")
@@ -25,4 +25,4 @@ dependencies {
api group: 'com.sun.xml.bind', name: 'jaxb-impl', version: jaxbVer
api group: 'com.sun.xml.bind', name: 'jaxb-core', version: '2.3.0'
}
-
+
