This is an automated email from the ASF dual-hosted git repository.
dspavlov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
The following commit(s) were added to refs/heads/master by this push:
new f2714a2e IGNITE-28954 Isolate cleaner tasks and reduce disk retention
(#242)
f2714a2e is described below
commit f2714a2e9aa5adfc8e806287075b62e938abd3b0
Author: Dmitriy Pavlov <[email protected]>
AuthorDate: Wed Aug 5 19:35:23 2026 +0300
IGNITE-28954 Isolate cleaner tasks and reduce disk retention (#242)
---
conf/branches.json | 4 +-
.../java/org/apache/ignite/ci/db/DbMigrations.java | 10 +-
.../org/apache/ignite/ci/db/Ignite2Configurer.java | 2 +-
jetty-launcher/build.gradle | 2 +-
jetty-launcher/src/main/resources/logback.xml | 2 +-
.../apache/ignite/migrate/GridIntListMigrator.java | 3 +-
.../ignite/migrate/GridIntListMigratorTest.java | 24 +++++
.../ignite/tcbot/engine/cleaner/Cleaner.java | 107 +++++++++++++++++++--
.../ignite/tcbot/engine/conf/CleanerConfig.java | 13 ++-
9 files changed, 142 insertions(+), 25 deletions(-)
diff --git a/conf/branches.json b/conf/branches.json
index ab5c6553..e0c426d7 100644
--- a/conf/branches.json
+++ b/conf/branches.json
@@ -9,8 +9,8 @@
"confidence": 0.995,
"cleanerConfig": {
"numOfItemsToDel": 100000,
- "safeDaysForCaches": 150,
- "safeDaysForLogs": 90,
+ "safeDaysForCaches": 110,
+ "safeDaysForLogs": 30,
"period": 1440
},
/* Teamcity Integration Parameters */
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
index 9b46cdeb..477d087d 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java
@@ -233,7 +233,8 @@ public class DbMigrations {
applyDestroyCacheMigration(Old.TEST_HIST_CACHE_NAME_V2_0);
applyDestroyCacheMigration(Old.SUITE_HIST_CACHE_NAME_V2_0);
- applyGridIntListMigration();
+ applyGridIntListMigration("migrate-GridIntList", null);
+ applyGridIntListMigration("migrate-GridIntList-botDetectedDefects",
"botDetectedDefects");
int sizeAfter = doneMigrations.size();
return (sizeAfter - sizeBefore) + " Migrations done from " + sizeAfter;
@@ -314,12 +315,11 @@ public class DbMigrations {
/**
* Applies the GridIntList migration from ignite.internal to tcbot-common
realization
*/
- private void applyGridIntListMigration() {
- applyMigration("migrate-GridIntList", () -> {
+ private void applyGridIntListMigration(String migrationCode, String
cacheFilter) {
+ applyMigration(migrationCode, () -> {
try {
- logger.info("Starting GridIntList type migration");
+ logger.info("Starting GridIntList type migration
[cacheFilter={}]", cacheFilter);
- String cacheFilter = null;
boolean apply = true;
boolean verbose = false;
int reportEvery = 50000;
diff --git
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/Ignite2Configurer.java
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/Ignite2Configurer.java
index 5e544161..572c2240 100644
---
a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/Ignite2Configurer.java
+++
b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/Ignite2Configurer.java
@@ -50,7 +50,7 @@ public class Ignite2Configurer {
public static final String LOG_TOTAL_SIZE_CAP =
"teamcity.bot.log.totalSizeCap";
/** Default total size cap for rolled logs. */
- public static final String DEFAULT_LOG_TOTAL_SIZE_CAP = "10GB";
+ public static final String DEFAULT_LOG_TOTAL_SIZE_CAP = "5GB";
public static void configLogger(File workDir, String subdir) {
LoggerContext logCtx =
(LoggerContext)LoggerFactory.getILoggerFactory();
diff --git a/jetty-launcher/build.gradle b/jetty-launcher/build.gradle
index 6113b151..16d75951 100644
--- a/jetty-launcher/build.gradle
+++ b/jetty-launcher/build.gradle
@@ -26,7 +26,7 @@ application {
"-Djava.net.preferIPv4Stack=true",
"-DIGNITE_QUIET=false",
"-Dteamcity.bot.ignite.metricsLogFrequencyMs=60000",
-
"-Dteamcity.bot.log.totalSizeCap=10GB",
+
"-Dteamcity.bot.log.totalSizeCap=5GB",
"-server",
"-Xms5g",
"-Xmx24g",
diff --git a/jetty-launcher/src/main/resources/logback.xml
b/jetty-launcher/src/main/resources/logback.xml
index f149fb52..c9f4ca81 100644
--- a/jetty-launcher/src/main/resources/logback.xml
+++ b/jetty-launcher/src/main/resources/logback.xml
@@ -1,6 +1,6 @@
<configuration>
<property name="TCBOT_LOG_DIR"
value="${teamcity.helper.home:-../work}/tcbot_logs"/>
- <property name="TCBOT_LOG_TOTAL_SIZE_CAP"
value="${teamcity.bot.log.totalSizeCap:-10GB}"/>
+ <property name="TCBOT_LOG_TOTAL_SIZE_CAP"
value="${teamcity.bot.log.totalSizeCap:-5GB}"/>
<appender name="FILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${TCBOT_LOG_DIR}/logfile-.log</file>
diff --git
a/migrator/src/main/java/org/apache/ignite/migrate/GridIntListMigrator.java
b/migrator/src/main/java/org/apache/ignite/migrate/GridIntListMigrator.java
index c1b0632f..7c12b5db 100644
--- a/migrator/src/main/java/org/apache/ignite/migrate/GridIntListMigrator.java
+++ b/migrator/src/main/java/org/apache/ignite/migrate/GridIntListMigrator.java
@@ -78,7 +78,8 @@ public final class GridIntListMigrator {
private static final List<String> GRID_INT_LIST_CACHE_NAMES =
Collections.unmodifiableList(Arrays.asList(
"teamcityFatBuild",
"teamcityFatBuildType",
- "teamcitySuiteHistory"
+ "teamcitySuiteHistory",
+ "botDetectedDefects"
));
/**
diff --git
a/migrator/src/test/java/org/apache/ignite/migrate/GridIntListMigratorTest.java
b/migrator/src/test/java/org/apache/ignite/migrate/GridIntListMigratorTest.java
index 3f415896..d0aa30be 100644
---
a/migrator/src/test/java/org/apache/ignite/migrate/GridIntListMigratorTest.java
+++
b/migrator/src/test/java/org/apache/ignite/migrate/GridIntListMigratorTest.java
@@ -55,6 +55,9 @@ public class GridIntListMigratorTest {
/** Production cache that may contain GridIntList. */
private static final String FAT_BUILD = "teamcityFatBuild";
+ /** Defect cache containing nested fat builds. */
+ private static final String BOT_DETECTED_DEFECTS = "botDetectedDefects";
+
/** Cache from the production failure. */
private static final String BUILD_LOG_CHECK_RESULT = "buildLogCheckResult";
@@ -256,6 +259,27 @@ public class GridIntListMigratorTest {
verify(ignite, never()).cache(BUILD_LOG_CHECK_RESULT);
}
+ /**
+ * Checks that defect values with nested fat builds are included in the
default migration.
+ */
+ @Test public void migrationScansBotDetectedDefectsByDefault() {
+ Ignite ignite = mock(Ignite.class);
+ IgniteCache<Object, Object> rawCache = mock(IgniteCache.class);
+ IgniteCache<Object, Object> binCache = mock(IgniteCache.class);
+ QueryCursor<Cache.Entry<Object, Object>> cursor =
mock(QueryCursor.class);
+
+
when(ignite.cacheNames()).thenReturn(Collections.singleton(BOT_DETECTED_DEFECTS));
+ when(ignite.cache(BOT_DETECTED_DEFECTS)).thenReturn(rawCache);
+ when(rawCache.withKeepBinary()).thenReturn(binCache);
+ when(binCache.query(any(ScanQuery.class))).thenReturn(cursor);
+ when(cursor.iterator()).thenReturn(Collections.<Cache.Entry<Object,
Object>>emptyList().iterator());
+
+ long updated = GridIntListMigrator.migrateOnInstance(ignite, null,
true, false, 1);
+
+ assertTrue("No entries should be updated in an empty cache", updated
== 0);
+ verify(ignite).cache(BOT_DETECTED_DEFECTS);
+ }
+
/**
* Test cache entry.
*/
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
index 85260758..18b37bc2 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/cleaner/Cleaner.java
@@ -23,10 +23,12 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantLock;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildConditionDao;
@@ -59,6 +61,8 @@ public class Cleaner {
private final AtomicBoolean init = new AtomicBoolean();
private final AtomicBoolean maintenanceActionRegistered = new
AtomicBoolean();
+ private final ReentrantLock cacheCleanLock = new ReentrantLock();
+ private final ReentrantLock logCleanLock = new ReentrantLock();
@Inject private IIssuesStorage issuesStorage;
@Inject private FatBuildDao fatBuildDao;
@@ -79,8 +83,11 @@ public class Cleaner {
private ScheduledExecutorService executorService;
- /** Maintenance action name. */
- public static final String MAINTENANCE_ACTION_NAME = "Cleaner.clean";
+ /** Cache cleanup maintenance action name. */
+ public static final String CACHE_CLEAN_ACTION_NAME = "Cleaner.cleanCaches";
+
+ /** Log cleanup maintenance action name. */
+ public static final String LOG_CLEAN_ACTION_NAME = "Cleaner.cleanLogs";
@AutoProfiling
@MonitoredTask(name = "Clean old cache data and log files")
@@ -103,9 +110,7 @@ public class Cleaner {
logger.info("Some log files (numOfItemsToDel=" +
numOfItemsToDel + ") older than " + thresholdDateForLogs + " will be removed.");
CacheCleanResult cacheRes =
removeCacheEntries(thresholdDateForCaches, numOfItemsToDel);
-
LogCleanResult logRes = removeLogFiles(thresholdDateForLogs,
numOfItemsToDel);
-
String res = cacheRes.summary() + "; " + logRes.summary();
report(res);
@@ -127,6 +132,52 @@ public class Cleaner {
}
}
+ @AutoProfiling
+ @MonitoredTask(name = "Clean old cache data")
+ public String cleanCaches() {
+ if (!cfg.getCleanerConfig().enabled()) {
+ logger.info("Periodic cache clean disabled.");
+
+ return "Periodic cache clean disabled.";
+ }
+
+ int numOfItemsToDel = cfg.getCleanerConfig().numOfItemsToDel();
+ long safeDaysForCaches = cfg.getCleanerConfig().safeDaysForCaches();
+ ZonedDateTime thresholdDate =
ZonedDateTime.now().minusDays(safeDaysForCaches);
+
+ logger.info("Some data from caches (numOfItemsToDel=" +
numOfItemsToDel + ") older than "
+ + thresholdDate + " will be removed.");
+
+ String res = removeCacheEntries(thresholdDate,
numOfItemsToDel).summary();
+
+ report(res);
+
+ return res;
+ }
+
+ @AutoProfiling
+ @MonitoredTask(name = "Clean old log files")
+ public String cleanLogs() {
+ if (!cfg.getCleanerConfig().enabled()) {
+ logger.info("Periodic log clean disabled.");
+
+ return "Periodic log clean disabled.";
+ }
+
+ int numOfItemsToDel = cfg.getCleanerConfig().numOfItemsToDel();
+ long safeDaysForLogs = cfg.getCleanerConfig().safeDaysForLogs();
+ ZonedDateTime thresholdDate =
ZonedDateTime.now().minusDays(safeDaysForLogs);
+
+ logger.info("Some log files (numOfItemsToDel=" + numOfItemsToDel + ")
older than " + thresholdDate
+ + " will be removed.");
+
+ String res = removeLogFiles(thresholdDate, numOfItemsToDel).summary();
+
+ report(res);
+
+ return res;
+ }
+
private CacheCleanResult removeCacheEntries(ZonedDateTime thresholdDate,
int numOfItemsToDel) {
long thresholdEpochMilli = thresholdDate.toInstant().toEpochMilli();
@@ -389,19 +440,57 @@ public class Cleaner {
buildConditionDao.init();
fatBuildDao.init();
- executorService = Executors.newSingleThreadScheduledExecutor();
+ executorService = Executors.newScheduledThreadPool(2);
- executorService.scheduleAtFixedRate(() -> self.get().clean(), 5,
cfg.getCleanerConfig().period(),
+ executorService.scheduleAtFixedRate(
+ () -> runScheduledClean("log files", logCleanLock, () ->
self.get().cleanLogs()),
+ 5,
+ cfg.getCleanerConfig().period(),
TimeUnit.MINUTES);
+ executorService.scheduleAtFixedRate(
+ () -> runScheduledClean("cache data", cacheCleanLock, () ->
self.get().cleanCaches()),
+ 10,
+ cfg.getCleanerConfig().period(),
+ TimeUnit.MINUTES);
+ }
+ }
+
+ /** Runs a periodic cleaner exclusively without allowing one failure to
suppress subsequent executions. */
+ private void runScheduledClean(String target, ReentrantLock lock,
Callable<String> clean) {
+ try {
+ runExclusive(lock, clean);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ logger.info("Scheduled cleanup of " + target + " interrupted");
+ }
+ catch (Throwable e) {
+ logger.error("Scheduled cleanup of " + target + " failed: " +
e.getMessage(), e);
+ }
+ }
+
+ /** Serializes runs of the same cleaner before entering its monitored
invocation. */
+ private String runExclusive(ReentrantLock lock, Callable<String> clean)
throws Exception {
+ lock.lockInterruptibly();
+
+ try {
+ return clean.call();
+ }
+ finally {
+ lock.unlock();
}
}
/** Registers manual cleaner action for monitoring management UI. */
private void registerMaintenanceAction() {
if (maintenanceActionRegistered.compareAndSet(false, true)) {
- maintenanceActions.register(MAINTENANCE_ACTION_NAME,
- "Run cleaner for old cache data and log files",
- () -> self.get().clean());
+ maintenanceActions.register(CACHE_CLEAN_ACTION_NAME,
+ "Remove old build data from Ignite caches",
+ () -> runExclusive(cacheCleanLock, () ->
self.get().cleanCaches()));
+ maintenanceActions.register(LOG_CLEAN_ACTION_NAME,
+ "Remove old downloaded build logs and technical log files",
+ () -> runExclusive(logCleanLock, () ->
self.get().cleanLogs()));
}
}
diff --git
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/CleanerConfig.java
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/CleanerConfig.java
index 07677bd8..d766ba80 100644
---
a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/CleanerConfig.java
+++
b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/CleanerConfig.java
@@ -19,7 +19,10 @@ package org.apache.ignite.tcbot.engine.conf;
/** */
public class CleanerConfig implements ICleanerConfig{
/** */
- public static final int DEFAULT_SAVE_DAYS = 30 * 6;
+ public static final int DEFAULT_CACHE_SAVE_DAYS = 110;
+
+ /** Default retention period for downloaded TeamCity build logs. */
+ public static final int DEFAULT_LOG_SAVE_DAYS = 30;
/** */
public static final int DEFAULT_NUMBER_OF_ITEMS_TO_DELETE = 100_000;
@@ -45,8 +48,8 @@ public class CleanerConfig implements ICleanerConfig{
/** */
public static CleanerConfig getDefaultCleanerConfig() {
CleanerConfig cfg = new CleanerConfig();
- cfg.safeDaysForCaches = DEFAULT_SAVE_DAYS;
- cfg.safeDaysForLogs = DEFAULT_SAVE_DAYS;
+ cfg.safeDaysForCaches = DEFAULT_CACHE_SAVE_DAYS;
+ cfg.safeDaysForLogs = DEFAULT_LOG_SAVE_DAYS;
cfg.numOfItemsToDel = DEFAULT_NUMBER_OF_ITEMS_TO_DELETE;
cfg.enabled = true;
cfg.period = DEFAULT_PERIOD_MINUTES;
@@ -55,12 +58,12 @@ public class CleanerConfig implements ICleanerConfig{
/** */
public int safeDaysForCaches() {
- return safeDaysForCaches == null || safeDaysForCaches < 0 ?
DEFAULT_SAVE_DAYS : safeDaysForCaches;
+ return safeDaysForCaches == null || safeDaysForCaches < 0 ?
DEFAULT_CACHE_SAVE_DAYS : safeDaysForCaches;
}
/** */
public int safeDaysForLogs() {
- return safeDaysForLogs == null || safeDaysForLogs < 0 ?
DEFAULT_SAVE_DAYS : safeDaysForLogs;
+ return safeDaysForLogs == null || safeDaysForLogs < 0 ?
DEFAULT_LOG_SAVE_DAYS : safeDaysForLogs;
}
/** */
