This is an automated email from the ASF dual-hosted git repository. dspavlov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit f23d373c678e402f7c4c5ceb741eb5ea4590da18 Author: Popov Aleksandr <[email protected]> AuthorDate: Wed Apr 29 20:28:39 2026 +0300 IGNITE-21899 Migrate GridIntList usage --- build.gradle | 2 +- ignite-tc-helper-web/build.gradle | 3 +- .../java/org/apache/ignite/ci/db/DbMigrations.java | 35 ++ migrator/README.md | 59 +++ migrator/build.gradle | 17 + .../apache/ignite/migrate/GridIntListMigrator.java | 242 ++++++++++ .../org/apache/ignite/migrate/MigratorArgs.java | 68 +++ .../org/apache/ignite/migrate/TransformResult.java | 32 +- .../org/apache/ignite/migrate/Transformer.java | 282 +++++++++++ settings.gradle | 1 + .../ignite/tcbot/common/util/GridIntIterator.java | 25 +- .../ignite/tcbot/common/util/GridIntList.java | 523 +++++++++++++++++++++ tcbot-persistence/build.gradle | 2 + .../ignited/buildtype/ParametersCompacted.java | 2 +- .../ignited/fatbuild/StatisticsCompacted.java | 2 +- .../tcignited/build/ProactiveFatBuildSync.java | 2 +- .../ignite/tcignited/buildref/BuildRefDao.java | 2 +- 17 files changed, 1270 insertions(+), 29 deletions(-) diff --git a/build.gradle b/build.gradle index eefac876..86f017f2 100644 --- a/build.gradle +++ b/build.gradle @@ -50,7 +50,7 @@ allprojects { jettyVer = '9.4.12.v20180830' - ignVer = '2.14.0' + ignVer = '2.17.0' guavaVer = '26.0-jre' diff --git a/ignite-tc-helper-web/build.gradle b/ignite-tc-helper-web/build.gradle index ffbe6b89..90109273 100644 --- a/ignite-tc-helper-web/build.gradle +++ b/ignite-tc-helper-web/build.gradle @@ -20,7 +20,8 @@ apply plugin: 'war' // https://www.apache.org/legal/resolved.html#category-a dependencies { - compile (project(":tcbot-engine")); + compile (project(":tcbot-engine")); + implementation project(':migrator') compile group: 'org.apache.ignite', name: 'ignite-core', version: ignVer compile group: 'org.apache.ignite', name: 'ignite-slf4j', version: ignVer diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java index 6781920d..ac2296e4 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/db/DbMigrations.java @@ -31,6 +31,7 @@ import org.apache.ignite.tcservice.model.result.Build; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.ignite.migrate.GridIntListMigrator; import javax.cache.Cache; import java.util.HashMap; @@ -232,6 +233,8 @@ public class DbMigrations { applyDestroyCacheMigration(Old.TEST_HIST_CACHE_NAME_V2_0); applyDestroyCacheMigration(Old.SUITE_HIST_CACHE_NAME_V2_0); + applyGridIntListMigration(); + int sizeAfter = doneMigrations.size(); return (sizeAfter - sizeBefore) + " Migrations done from " + sizeAfter; @@ -308,6 +311,38 @@ public class DbMigrations { return ignite.getOrCreateCache(ccfg); } + /** + * Applies the GridIntList migration from ignite.internal to tcbot-common realization + */ + private void applyGridIntListMigration() { + applyMigration("migrate-GridIntList", () -> { + try { + logger.info("Starting GridIntList type migration"); + + String cacheFilter = null; + boolean apply = true; + boolean verbose = false; + int reportEvery = 500; + + long updated = GridIntListMigrator.migrateOnInstance( + ignite, + cacheFilter, + apply, + verbose, + reportEvery + ); + + logger.info("GridIntList migration completed. Updated {} entries", updated); + + } + catch (Exception e) { + logger.error("GridIntList migration failed", e); + + throw new RuntimeException("GridIntList migration failed", e); + } + }); + } + private void applyMigration(String code, Runnable runnable) { if (doneMigrations.containsKey(code)) return; diff --git a/migrator/README.md b/migrator/README.md new file mode 100644 index 00000000..cf0a946b --- /dev/null +++ b/migrator/README.md @@ -0,0 +1,59 @@ +# Migrator: Ignite internal API GridIntList → TCbot util GridIntList + +Purpose: +- Offline tool to migrate TeamCity Bot Ignite persistence (work/*), replacing legacy ```org.apache.ignite.internal.util.GridIntList``` with ```org.apache.ignite.tcbot.common.util.GridIntList``` +- Works directly on binary data (keepBinary), recursively transforms fields/collections/maps/arrays and rewrites only changed entries. +- Supports dry-run (no writes) and apply (write changes) modes. +- Safe to run on a copy of work/ directory and robust to partial errors (skips bad entries, logs warnings). + +Safety: +- Stop tcbot before migration. +- Always run on a copy of work/: never point at a live folder. +- Never run in parallel with tcbot or another migrator on the same persistence. + +Requirements: +- JDK 11+ +- Gradle wrapper from tcbot repo +- New GridIntList on classpath of migrator (org.apache.ignite.tcbot.common.util.GridIntList) + +Build: +- From repo root: + - ```./gradlew -p migrator clean build``` + +Quick start (macOS/Linux): +1) Backup work: + - ```cp -a </path/to/tcbot/work> </path/to/work_backup>``` +2) Dry-run with verbose report (no changes apply): + - ```export IGNITE_WORK_DIR=</path/to/work_backup>``` + - ```./gradlew -p migrator run --args="--verbose"``` +3) Apply (write changes): + - ```export IGNITE_WORK_DIR=</path/to/work_backup>``` + - ```./gradlew -p migrator run --args="--apply"``` + - Optional: focus on a cache: `````--cache <cacheName>````` + +CLI arguments: +- ```--apply``` write changes (otherwise dry-run) +- ```--verbose``` print extra diagnostics +- ```--cache <substr>``` process only caches whose name contains the substring +- ```--report <N>``` progress interval (log every N scanned entries) +- ```--workDir <path>``` path to work/ directory (overrides IGNITE_WORK_DIR) + +How it works: +- Scans caches with ScanQuery in keepBinary mode (values as BinaryObject). +- Recursively traverses value graphs: + - If a node is legacy GridIntList (BinaryObject or Java object) → extract int[] → build new GridIntList (fallback to int[] if class missing). + - For BinaryObject parents → rebuild with BinaryObjectBuilder only if any child changed. + - For List/Set/Map/Object[] → transform elements and rebuild container only if needed (stable order via LinkedHashSet/LinkedHashMap). +- Writes back only changed entries. + +What it changes / does not change: +- Changes: any occurrence of legacy GridIntList in values (fields, collections, arrays) is replaced with the new type (or int[] fallback). +- Does NOT change: cache keys, unrelated fields/types, untouched caches. + +Verification: +- Migrator logs per-cache summary: ```scanned=``` ```updated=``` + +Notes: +- The migrator guards recursion (MAX_DEPTH=32). It logs and skips overly deep branches to stay robust. MAX_DEPTH is +eligible to be tuned due specific cache. +- The tool logs with SLF4J (console). Deteministic iteration order is preserved for Set/Map to keep binary output stable. \ No newline at end of file diff --git a/migrator/build.gradle b/migrator/build.gradle new file mode 100644 index 00000000..937af99e --- /dev/null +++ b/migrator/build.gradle @@ -0,0 +1,17 @@ +plugins { + id 'application' +} + +repositories { + mavenCentral() +} + +dependencies { + implementation 'org.apache.ignite:ignite-core:2.16.0' + implementation 'ch.qos.logback:logback-classic:1.2.3' + implementation project(':tcbot-common') +} + +application { + mainClass = 'org.apache.ignite.migrate.GridIntListMigrator' +} \ No newline at end of file diff --git a/migrator/src/main/java/org/apache/ignite/migrate/GridIntListMigrator.java b/migrator/src/main/java/org/apache/ignite/migrate/GridIntListMigrator.java new file mode 100644 index 00000000..14f5bd24 --- /dev/null +++ b/migrator/src/main/java/org/apache/ignite/migrate/GridIntListMigrator.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.migrate; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.ScanQuery; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.configuration.BinaryConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.cache.Cache; +import java.io.File; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Offline migrator for TeamCity Bot Ignite persistence. + * <p> + * Recursively scans all entries in Ignite caches and replaces any occurrence of the legacy type + * org.apache.ignite.internal.util.GridIntList with the new type + * org.apache.ignite.tcbot.common.util.GridIntList, preserving the int[] payload. + * <p> + * Usage: + * export IGNITE_WORK_DIR=/abs/path/to/work_backup + * ./gradlew -p migrator run --args="--verbose --report 200" # dry run with verbose report + * ./gradlew -p migrator run --args="--apply --report 500" # apply to all caches + */ + +public final class GridIntListMigrator { + /** + * Logger. + */ + private static final Logger log = LoggerFactory.getLogger(GridIntListMigrator.class); + + /** + * Scan page size. + */ + private static final int DEFAULT_PAGE_SIZE = 256; + + /** + * Default constructor. + */ + private GridIntListMigrator() { + } + + /** + * Boots a standalone Ignite node on the given work dir and executes migration. + */ + public static void main(String[] args) { + System.setProperty("java.net.preferIPv4Stack", "true"); + + MigratorArgs a = MigratorArgs.parse(args); + + if (a.workDir == null || a.workDir.isEmpty()) { + String wd = System.getProperty("IGNITE_WORK_DIR"); + + if (wd == null || wd.isEmpty()) + wd = System.getenv("IGNITE_WORK_DIR"); + + a.workDir = wd; + } + + if (a.workDir == null || a.workDir.isEmpty()) { + log.error("IGNITE_WORK_DIR is not set. Use --workDir or set system/env variable."); + + System.exit(2); + } + + File checkDir = new File(a.workDir); + + if (!checkDir.isDirectory()) { + log.error("IGNITE_WORK_DIR is not a directory: {}", checkDir.getAbsolutePath()); + + System.exit(2); + } + + IgniteConfiguration cfg = new IgniteConfiguration() + .setIgniteInstanceName("tcbot-migrator") + .setWorkDirectory(a.workDir); + + String consistentId = detectConsistentId(a.workDir); + + if (consistentId != null) { + cfg.setConsistentId(consistentId); + + log.info("Using consistentId={}", consistentId); + } + else + log.warn("Couldnt detect consistentId."); + + DataStorageConfiguration ds = new DataStorageConfiguration(); + ds.getDefaultDataRegionConfiguration().setPersistenceEnabled(true); + cfg.setDataStorageConfiguration(ds); + + BinaryConfiguration bcfg = new BinaryConfiguration(); + cfg.setBinaryConfiguration(bcfg); + + Ignition.setClientMode(false); + + try (Ignite ig = Ignition.start(cfg)) { + ig.cluster().state(ClusterState.ACTIVE); + + long updated = migrateOnInstance( + ig, + a.cacheFilter, + a.apply, + a.verbose, + a.reportEvery + ); + + log.info("Migration finished. Total updated: {}", updated); + } + } + + /** + * Perform migration on existing Ignite instance + * @param ignite Ignite instance + * @param cacheFilter cache name filter + * @param apply true to apply changes, false to dry-run + * @param verbose logging + * @param reportEvery frequency of reports + * @return number of updated records + */ + public static long migrateOnInstance(Ignite ignite, + String cacheFilter, + boolean apply, + boolean verbose, + int reportEvery) { + Collection<String> cacheNames = new ArrayList<>(ignite.cacheNames()); + + if (cacheFilter != null && !cacheFilter.isEmpty()) + cacheNames.removeIf(n -> !n.contains(cacheFilter)); + + log.info("GridIntList migration - Caches to scan: {}", cacheNames); + + Transformer transformer = new Transformer(verbose); + long totalUpdated = 0; + + for (String cacheName : cacheNames) { + IgniteCache<Object, Object> rawCache = ignite.cache(cacheName); + + if (rawCache == null) + continue; + + IgniteCache<Object, Object> c = rawCache.withKeepBinary(); + + log.info("GridIntList migration - Scanning cache: {}", cacheName); + + ScanQuery<Object, Object> q = new ScanQuery<>(); + q.setPageSize(DEFAULT_PAGE_SIZE); + + AtomicLong scanned = new AtomicLong(); + AtomicLong updated = new AtomicLong(); + + try (QueryCursor<Cache.Entry<Object, Object>> cur = c.query(q)) { + for (Cache.Entry<Object, Object> e : cur) { + try { + Object v = e.getValue(); + TransformResult tr = transformer.transform(v, 0); + + if (tr.changed) { + if (apply) { + c.put(e.getKey(), tr.val); + + updated.incrementAndGet(); + } + else if (verbose) + log.info("DRY-RUN would update key={}", e.getKey()); + } + + long s = scanned.incrementAndGet(); + + if (s % reportEvery == 0) + log.info("Scanned={} updated={}", s, updated.get()); + + } + catch (Throwable t) { + log.warn("Entry migration failed, skipping. Cause: {}", t.toString()); + + scanned.incrementAndGet(); + } + } + } + + log.info("Done {}: scanned={} updated={}", cacheName, scanned.get(), updated.get()); + + totalUpdated += updated.get(); + } + + log.info("GridIntList migration finished. Total updated: {}", totalUpdated); + + return totalUpdated; + } + + /** + * Infers consistentId by reading first subdirectory under work/db. + * + * @param workDir path to Ignite work dir. + * @return consistentId or null if not found. + */ + private static String detectConsistentId(String workDir) { + File dbDir = new File(workDir, "db"); + + if (!dbDir.isDirectory()) + return null; + + File[] kids = dbDir.listFiles(File::isDirectory); + + if (kids == null || kids.length == 0) + return null; + + return kids[0].getName(); + } + + /** + * Class static logger. + */ + public static Logger GetMigratorLogger() { + return log; + } +} \ No newline at end of file diff --git a/migrator/src/main/java/org/apache/ignite/migrate/MigratorArgs.java b/migrator/src/main/java/org/apache/ignite/migrate/MigratorArgs.java new file mode 100644 index 00000000..97447636 --- /dev/null +++ b/migrator/src/main/java/org/apache/ignite/migrate/MigratorArgs.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.migrate; + +/** + * Migrator command-line arguments. + * <p> + * Supported flags: + * --apply actually write changes (otherwise dry-run) + * --verbose more diagnostics + * --cache <substr> process only caches whose name contains given substring + * --report <N> progress log interval + * --workDir <path> path to work/ directory (overrides IGNITE_WORK_DIR) + */ +public final class MigratorArgs { + boolean apply = false; + boolean verbose = false; + String cacheFilter = null; + int reportEvery = 500; + String workDir = null; + + static MigratorArgs parse(String[] args) { + MigratorArgs cliArgs = new MigratorArgs(); + + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "--apply": + cliArgs.apply = true; + break; + case "--verbose": + cliArgs.verbose = true; + break; + case "--cache": + cliArgs.cacheFilter = args[++i]; + break; + case "--report": + cliArgs.reportEvery = Integer.parseInt(args[++i]); + break; + case "--workDir": + cliArgs.workDir = args[++i]; + break; + default: + break; + } + } + GridIntListMigrator.GetMigratorLogger().info( + "Args: apply={} verbose={} cacheFilter={} reportEvery={} workDir={}", + cliArgs.apply, cliArgs.verbose, cliArgs.cacheFilter, cliArgs.reportEvery, cliArgs.workDir + ); + + return cliArgs; + } +} \ No newline at end of file diff --git a/tcbot-persistence/build.gradle b/migrator/src/main/java/org/apache/ignite/migrate/TransformResult.java similarity index 59% copy from tcbot-persistence/build.gradle copy to migrator/src/main/java/org/apache/ignite/migrate/TransformResult.java index c6118a84..31170e76 100644 --- a/tcbot-persistence/build.gradle +++ b/migrator/src/main/java/org/apache/ignite/migrate/TransformResult.java @@ -15,18 +15,28 @@ * limitations under the License. */ -apply plugin: 'java' +package org.apache.ignite.migrate; -repositories { - mavenCentral() - mavenLocal() -} +/** + * Result of a recursive transformation: value and flag indicating changes. + */ +public final class TransformResult { + /** Value. */ + final Object val; + + /** Changed. */ + final boolean changed; -dependencies { - compile (project(":tcbot-common")); + private TransformResult(Object val, boolean changed) { + this.val = val; + this.changed = changed; + } + + static TransformResult same(Object v) { + return new TransformResult(v, false); + } - compile (group: 'org.apache.ignite', name: 'ignite-core', version: ignVer) { - exclude group: 'org.jetbrains' + static TransformResult changed(Object v) { + return new TransformResult(v, true); } -} - \ No newline at end of file +} \ No newline at end of file diff --git a/migrator/src/main/java/org/apache/ignite/migrate/Transformer.java b/migrator/src/main/java/org/apache/ignite/migrate/Transformer.java new file mode 100644 index 00000000..eedc142a --- /dev/null +++ b/migrator/src/main/java/org/apache/ignite/migrate/Transformer.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.migrate; + +import java.lang.reflect.Constructor; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.binary.BinaryObjectBuilder; +import org.slf4j.Logger; + +/** + * Performs recursive transformation of values, replacing legacy GridIntList with the new type. + * Guarded by a MAX_DEPTH to avoid accidental cycles. + */ +public final class Transformer { + /** CLI verbose flag. */ + private final boolean verbose; + + /** + * Fully-qualified name of the legacy type to replace. + */ + private static final String OLD_KEYS_TYPE = "org.apache.ignite.internal.util.GridIntList"; + + /** + * Fully-qualified name of the new type. + */ + private static final String NEW_KEYS_TYPE = "org.apache.ignite.tcbot.common.util.GridIntList"; + + /** + * Recursion guard to prevent accidental cycles. + */ + private static final int MAX_DEPTH = 32; + + /** + * Logger. + */ + private static final Logger log = GridIntListMigrator.GetMigratorLogger(); + + /** Cached constructor of NEW_KEYS_TYPE(int[]) to avoid excessive reflection. */ + private final Constructor<?> newKeysCachedConstruct; + + Transformer(boolean verb) { + verbose = verb; + newKeysCachedConstruct = findNewKeysConstruct(); + } + + /** + * Recursively transforms a value. + * <p> + * Rules: + * - BinaryObject of OLD_KEYS_TYPE -> build NEW_KEYS_TYPE or fallback to int[]. + * - Java object of OLD_KEYS_TYPE -> same replacement. + * - BinaryObject (other type) -> rebuild only if some field changed. + * - List/Set/Map/Object[] -> rebuild container only if any element changed. + * - Primitives/String/int[] -> unchanged. + * + * @param v value to transform + * @param depth recursion depth (fixed guard inside) + * @return transform result with possibly new value and 'changed' flag + */ + TransformResult transform(Object v, int depth) { + if (v == null) + return TransformResult.same(null); + + // Strict migration interruption + if (depth > MAX_DEPTH) { + String errMsg = String.format("Max depth %d reached; value type=%s, aborting migration.", MAX_DEPTH, v.getClass()); + + throw new IllegalStateException(errMsg); + } + + // Binary legacy type + if (v instanceof BinaryObject) { + BinaryObject bo = (BinaryObject)v; + String typeName = bo.type().typeName(); + + if (typeName.equals(OLD_KEYS_TYPE)) { + int[] ints = extractInts(bo); + Object newKeys = buildNewKeys(ints); + + return TransformResult.changed(newKeys); + } + + // Generic BinaryObject + BinaryObjectBuilder bb = bo.toBuilder(); + boolean anyChanged = false; + + for (String oldChildField : bo.type().fieldNames()) { + // Transform nested fields + TransformResult childRes = transform(bo.field(oldChildField), depth + 1); + + if (childRes.changed) { + bb.setField(oldChildField, childRes.val); + + anyChanged = true; + } + } + + // Rebuild only if some field changed + return anyChanged ? TransformResult.changed(bb.build()) : TransformResult.same(v); + } + + // Java legacy type + if (v instanceof org.apache.ignite.internal.util.GridIntList) { + org.apache.ignite.internal.util.GridIntList g = (org.apache.ignite.internal.util.GridIntList)v; + int[] ints = new int[g.size()]; + + for (int i = 0; i < ints.length; i++) + ints[i] = g.get(i); + + Object newKeys = buildNewKeys(ints); + + return TransformResult.changed(newKeys); + } + + // Already new type + if (v.getClass().getName().equals(NEW_KEYS_TYPE)) + return TransformResult.same(v); + + // Collections + if (v instanceof List) { + List<?> src = (List<?>)v; + boolean anyChanged = false; + + List<Object> out = new ArrayList<>(src.size()); + + for (Object listElem : src) { + TransformResult tr = transform(listElem, depth + 1); + + out.add(tr.val); + anyChanged |= tr.changed; + } + + return anyChanged ? TransformResult.changed(out) : TransformResult.same(v); + } + + if (v instanceof Set) { + Set<?> src = (Set<?>)v; + boolean anyChanged = false; + + LinkedHashSet<Object> out = new LinkedHashSet<>(Math.max(16, (int)Math.ceil(src.size() / 0.75))); + + for (Object el : src) { + TransformResult tr = transform(el, depth + 1); + + out.add(tr.val); + anyChanged |= tr.changed; + } + + return anyChanged ? TransformResult.changed(out) : TransformResult.same(v); + } + + if (v instanceof Map) { + Map<?, ?> src = (Map<?, ?>)v; + boolean anyChanged = false; + + LinkedHashMap<Object, Object> out = new LinkedHashMap<>(Math.max(16, (int)Math.ceil(src.size() / 0.75))); + + for (Map.Entry<?, ?> en : src.entrySet()) { + TransformResult k = transform(en.getKey(), depth + 1); + TransformResult val = transform(en.getValue(), depth + 1); + + out.put(k.val, val.val); + anyChanged |= k.changed || val.changed; + } + + return anyChanged ? TransformResult.changed(out) : TransformResult.same(v); + } + + // Object[] arrays (non-primitive) + if (v.getClass().isArray() && !v.getClass().getComponentType().isPrimitive()) { + Object[] arr = (Object[])v; + Object[] out = new Object[arr.length]; + + boolean anyChanged = false; + + for (int i = 0; i < arr.length; i++) { + TransformResult tr = transform(arr[i], depth + 1); + + out[i] = tr.val; + anyChanged |= tr.changed; + } + + return anyChanged ? TransformResult.changed(out) : TransformResult.same(v); + } + + // Primitives, String, int[] etc.: unchanged + return TransformResult.same(v); + } + + /** + * Extracts int[] from legacy GridIntList BinaryObject. + */ + private int[] extractInts(BinaryObject oldKeys) { + try { + Object obj = oldKeys.deserialize(); + + if (obj instanceof org.apache.ignite.internal.util.GridIntList) { + org.apache.ignite.internal.util.GridIntList g = (org.apache.ignite.internal.util.GridIntList)obj; + + return g.array(); + } + } + catch (Throwable t) { + if (verbose) + log.info("Deserialize fallback: {}", t.toString()); + } + + // Hardcode ("arr") based on GridIntList fields + Collection<String> childFields = oldKeys.type().fieldNames(); + if (childFields.contains("arr")) { + int[] arr = oldKeys.field("arr"); + + if (arr != null) + return arr; + } + + log.warn("Can't extract ints from {} fields={}", oldKeys.type().typeName(), childFields); + + return new int[0]; // best effort fallback + } + + /** + * Builds an instance of the new GridIntList (org.apache.ignite.tcbot.common.util.GridIntList), + * or falls back to int[] if the class is not on the classpath. + */ + private Object buildNewKeys(int[] ints) { + if (newKeysCachedConstruct != null) { + try { + return newKeysCachedConstruct.newInstance((Object)ints); + } + catch (ReflectiveOperationException ignored) { + // fall through to fallback + } + } + + if (verbose) + log.warn("NEW_KEYS_TYPE {} is not available; falling back to raw int[]", NEW_KEYS_TYPE); + + return ints; // best effort fallback + } + + /** + * Resolves constructor NEW_KEYS_TYPE(int[]) once to avoid reflection per entry. + */ + private Constructor<?> findNewKeysConstruct() { + try { + Class<?> cls = Class.forName(NEW_KEYS_TYPE); + + return cls.getConstructor(int[].class); + } + catch (Throwable t) { + if (verbose) { + log.warn("New keys type {} is not on classpath, will fallback to int[] (cause: {})", + NEW_KEYS_TYPE, t.toString()); + } + + return null; + } + } +} diff --git a/settings.gradle b/settings.gradle index 1baddb33..fd67a803 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,6 @@ include 'ignite-tc-helper-web' include 'jetty-launcher' +include 'migrator' include 'tcbot-server-node' include 'tcbot-common' include 'tcbot-notify' diff --git a/tcbot-persistence/build.gradle b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/util/GridIntIterator.java similarity index 72% copy from tcbot-persistence/build.gradle copy to tcbot-common/src/main/java/org/apache/ignite/tcbot/common/util/GridIntIterator.java index c6118a84..35f1cd63 100644 --- a/tcbot-persistence/build.gradle +++ b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/util/GridIntIterator.java @@ -15,18 +15,19 @@ * limitations under the License. */ -apply plugin: 'java' +package org.apache.ignite.tcbot.common.util; -repositories { - mavenCentral() - mavenLocal() -} - -dependencies { - compile (project(":tcbot-common")); +/** + * Iterator over integer primitives. + */ +public interface GridIntIterator { + /** + * @return {@code true} if the iteration has more elements. + */ + public boolean hasNext(); - compile (group: 'org.apache.ignite', name: 'ignite-core', version: ignVer) { - exclude group: 'org.jetbrains' - } + /** + * @return Next int. + */ + public int next(); } - \ No newline at end of file diff --git a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/util/GridIntList.java b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/util/GridIntList.java new file mode 100644 index 00000000..2d4e43c5 --- /dev/null +++ b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/util/GridIntList.java @@ -0,0 +1,523 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.common.util; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Arrays; +import java.util.NoSuchElementException; +import javax.annotation.Nullable; + +/** + * Minimal list API to work with primitive ints. This list exists + * to avoid boxing/unboxing when using standard list from Java. + */ +public class GridIntList implements Externalizable { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private int[] arr; + + /** */ + private int idx; + + /** + * + */ + public GridIntList() { + // No-op. + } + + /** + * @param size Size. + */ + public GridIntList(int size) { + arr = new int[size]; + // idx = 0 + } + + /** + * @param arr Array. + */ + public GridIntList(int[] arr) { + this.arr = arr.clone(); + + idx = arr.length; + } + + /** + * @param vals Values. + * @return List from values. + */ + public static GridIntList asList(int... vals) { + if (vals == null || vals.length == 0) + return new GridIntList(); + + return new GridIntList(vals); + } + + /** + * @param arr Array. + * @param size Size. + */ + private GridIntList(int[] arr, int size) { + this.arr = arr; + idx = size; + } + + /** + * @return Copy of this list. + */ + public GridIntList copy() { + if (idx == 0) + return new GridIntList(); + + return new GridIntList(Arrays.copyOf(arr, idx)); + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (!(o instanceof GridIntList)) + return false; + + GridIntList that = (GridIntList)o; + + if (idx != that.idx) + return false; + + if (idx == 0 || arr == that.arr) + return true; + + for (int i = 0; i < idx; i++) { + if (arr[i] != that.arr[i]) + return false; + } + + return true; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = 1; + + for (int i = 0; i < idx; i++) { + int element = arr[i]; + res = 31 * res + element; + } + + return res; + } + + /** + * @param l List to add all elements of. + */ + public void addAll(GridIntList l) { + assert l != null; + + if (l.isEmpty()) + return; + + if (arr == null) + arr = new int[4]; + + int len = arr.length; + + while (len < idx + l.size()) + len <<= 1; + + arr = Arrays.copyOf(arr, len); + + System.arraycopy(l.arr, 0, arr, idx, l.size()); + + idx += l.size(); + } + + /** + * Add element to this array. + * @param x Value. + */ + public void add(int x) { + if (arr == null) + arr = new int[4]; + else if (arr.length == idx) + arr = Arrays.copyOf(arr, arr.length << 1); + + arr[idx++] = x; + } + + /** + * Clears the list. + */ + public void clear() { + idx = 0; + } + + /** + * Gets the last element. + * + * @return The last element. + */ + public int last() { + return arr[idx - 1]; + } + + /** + * Removes and returns the last element of the list. Complementary method to {@link #add(int)} for stack like usage. + * + * @return Removed element. + * @throws NoSuchElementException If the list is empty. + */ + public int remove() throws NoSuchElementException { + if (idx == 0) + throw new NoSuchElementException(); + + return arr[--idx]; + } + + /** + * Returns (possibly reordered) copy of this list, excluding all elements of given list. + * + * @param l List of elements to remove. + * @return New list without all elements from {@code l}. + */ + public GridIntList copyWithout(GridIntList l) { + assert l != null; + + if (idx == 0) + return new GridIntList(); + + if (l.idx == 0) + return new GridIntList(Arrays.copyOf(arr, idx)); + + int[] newArr = Arrays.copyOf(arr, idx); + int newIdx = idx; + + for (int i = 0; i < l.size(); i++) { + int rmVal = l.get(i); + + for (int j = 0; j < newIdx; j++) { + if (newArr[j] == rmVal) { + + while (newIdx > 0 && newArr[newIdx - 1] == rmVal) + newIdx--; + + if (newIdx > 0) { + newArr[j] = newArr[newIdx - 1]; + newIdx--; + } + } + } + } + + return new GridIntList(newArr, newIdx); + } + + /** + * @param i Index. + * @return Value. + */ + public int get(int i) { + assert i < idx; + + return arr[i]; + } + + /** + * @return Size. + */ + public int size() { + return idx; + } + + /** + * @return {@code True} if this list has no elements. + */ + public boolean isEmpty() { + return idx == 0; + } + + /** + * @param l Element to find. + * @return {@code True} if found. + */ + public boolean contains(int l) { + for (int i = 0; i < idx; i++) { + if (arr[i] == l) + return true; + } + + return false; + } + + /** + * @param l List to check. + * @return {@code True} if this list contains all the elements of passed in list. + */ + public boolean containsAll(GridIntList l) { + for (int i = 0; i < l.size(); i++) { + if (!contains(l.get(i))) + return false; + } + + return true; + } + + /** + * @return {@code True} if there are no duplicates. + */ + public boolean distinct() { + for (int i = 0; i < idx; i++) { + for (int j = i + 1; j < idx; j++) { + if (arr[i] == arr[j]) + return false; + } + } + + return true; + } + + /** + * @param size New size. + * @param last If {@code true} the last elements will be removed, otherwise the first. + */ + public void truncate(int size, boolean last) { + assert size >= 0 && size <= idx; + + if (size == idx) + return; + + if (!last && idx != 0 && size != 0) + System.arraycopy(arr, idx - size, arr, 0, size); + + idx = size; + } + + /** + * Removes element by given index. + * + * @param i Index. + * @return Removed value. + */ + public int removeIndex(int i) { + assert i < idx : i; + + int res = arr[i]; + + if (i == idx - 1) { // Last element. + idx = i; + } + else { + System.arraycopy(arr, i + 1, arr, i, idx - i - 1); + idx--; + } + + return res; + } + + /** + * Removes value from this list. + * + * @param startIdx Index to begin search with. + * @param val Value. + * @return Index of removed value if the value was found and removed or {@code -1} otherwise. + */ + public int removeValue(int startIdx, int val) { + assert startIdx >= 0; + + for (int i = startIdx; i < idx; i++) { + if (arr[i] == val) { + removeIndex(i); + + return i; + } + } + + return -1; + } + + /** + * Removes value from this list. + * + * @param startIdx Index to begin search with. + * @param oldVal Old value. + * @param newVal New value. + * @return Index of replaced value if the value was found and replaced or {@code -1} otherwise. + */ + public int replaceValue(int startIdx, int oldVal, int newVal) { + for (int i = startIdx; i < idx; i++) { + if (arr[i] == oldVal) { + arr[i] = newVal; + + return i; + } + } + + return -1; + } + + /** + * @return Array copy. + */ + public int[] array() { + int[] res = new int[idx]; + + System.arraycopy(arr, 0, res, 0, idx); + + return res; + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeInt(idx); + + for (int i = 0; i < idx; i++) + out.writeInt(arr[i]); + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException { + idx = in.readInt(); + + arr = new int[idx]; + + for (int i = 0; i < idx; i++) + arr[i] = in.readInt(); + } + + /** {@inheritDoc} */ + @Override public String toString() { + StringBuilder b = new StringBuilder("["); + + for (int i = 0; i < idx; i++) { + if (i != 0) + b.append(','); + + b.append(arr[i]); + } + + b.append(']'); + return b.toString(); + } + + /** + * @param in Input to read list from. + * @return Grid int list. + * @throws IOException If failed. + */ + @Nullable + public static GridIntList readFrom(DataInput in) throws IOException { + int idx = in.readInt(); + + if (idx == -1) + return null; + + int[] arr = new int[idx]; + + for (int i = 0; i < idx; i++) + arr[i] = in.readInt(); + + return new GridIntList(arr); + } + + /** + * @param out Output to write to. + * @param list List. + * @throws IOException If failed. + */ + public static void writeTo(DataOutput out, @Nullable GridIntList list) throws IOException { + out.writeInt(list != null ? list.idx : -1); + + if (list != null) { + for (int i = 0; i < list.idx; i++) + out.writeInt(list.arr[i]); + } + } + + /** + * @param to To list. + * @param from From list. + * @return To list (passed in or created). + */ + public static GridIntList addAll(@Nullable GridIntList to, GridIntList from) { + if (to == null) { + GridIntList res = new GridIntList(from.size()); + + res.addAll(from); + + return res; + } + else { + to.addAll(from); + + return to; + } + } + + /** + * Sorts this list. + * Use {@code copy().sort()} if you need a defensive copy. + * + * @return {@code this} For chaining. + */ + public GridIntList sort() { + if (idx > 1) + Arrays.sort(arr, 0, idx); + + return this; + } + + /** + * Removes given number of elements from the end. If the given number of elements is higher than + * list size, then list will be cleared. + * + * @param cnt Count to pop from the end. + */ + public void pop(int cnt) { + assert cnt >= 0 : cnt; + + if (idx < cnt) + idx = 0; + else + idx -= cnt; + } + + /** + * @return Iterator. + */ + public GridIntIterator iterator() { + return new GridIntIterator() { + int c; + + @Override public boolean hasNext() { + return c < idx; + } + + @Override public int next() { + return arr[c++]; + } + }; + } +} diff --git a/tcbot-persistence/build.gradle b/tcbot-persistence/build.gradle index c6118a84..82c2e63f 100644 --- a/tcbot-persistence/build.gradle +++ b/tcbot-persistence/build.gradle @@ -28,5 +28,7 @@ dependencies { compile (group: 'org.apache.ignite', name: 'ignite-core', version: ignVer) { exclude group: 'org.jetbrains' } + + implementation "org.apache.ignite:ignite-indexing:$ignVer" } \ No newline at end of file diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildtype/ParametersCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildtype/ParametersCompacted.java index b889c485..65ac3a10 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildtype/ParametersCompacted.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/buildtype/ParametersCompacted.java @@ -19,7 +19,7 @@ package org.apache.ignite.ci.teamcity.ignited.buildtype; import com.google.common.base.MoreObjects; import com.google.common.base.Strings; -import org.apache.ignite.internal.util.GridIntList; +import org.apache.ignite.tcbot.common.util.GridIntList; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcbot.persistence.Persisted; import org.apache.ignite.tcservice.model.conf.bt.Parameters; diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java index 514542b5..fd021a9b 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/StatisticsCompacted.java @@ -24,7 +24,7 @@ import org.apache.ignite.tcbot.persistence.Persisted; import org.apache.ignite.tcservice.model.conf.bt.Property; import org.apache.ignite.tcservice.model.result.stat.Statistics; import org.apache.ignite.tcbot.persistence.IStringCompactor; -import org.apache.ignite.internal.util.GridIntList; +import org.apache.ignite.tcbot.common.util.GridIntList; import org.apache.ignite.internal.util.GridLongList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java index 489ef59c..09544dd0 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/build/ProactiveFatBuildSync.java @@ -40,7 +40,7 @@ import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; import org.apache.ignite.ci.teamcity.ignited.change.ChangeSync; import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; import org.apache.ignite.internal.util.GridConcurrentHashSet; -import org.apache.ignite.internal.util.GridIntList; +import org.apache.ignite.tcbot.common.util.GridIntList; import org.apache.ignite.tcbot.common.exeption.ExceptionUtil; import org.apache.ignite.tcbot.common.exeption.ServiceConflictException; import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java index b8fbd1e6..771678e6 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/buildref/BuildRefDao.java @@ -45,7 +45,7 @@ import org.apache.ignite.cache.query.SqlQuery; import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted; import org.apache.ignite.ci.teamcity.ignited.runhist.RunHistKey; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.internal.util.GridIntList; +import org.apache.ignite.tcbot.common.util.GridIntList; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.tcbot.common.conf.TcBotSystemProperties; import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
