[ignite-3] branch main updated: IGNITE-18746 Sql. Sargs implementation alignment - Fixes #1721.
This is an automated email from the ASF dual-hosted git repository. zstan pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/ignite-3.git The following commit(s) were added to refs/heads/main by this push: new d53663e52c IGNITE-18746 Sql. Sargs implementation alignment - Fixes #1721. d53663e52c is described below commit d53663e52cc2b69af92abce6afad9ee3d81ffdf2 Author: zstan AuthorDate: Tue Mar 7 10:03:44 2023 +0300 IGNITE-18746 Sql. Sargs implementation alignment - Fixes #1721. Signed-off-by: zstan --- modules/rest-api/openapi/openapi.yaml | 3 + .../sql/engine/ItHashSpoolIntegrationTest.java | 4 +- .../ignite/internal/sql/engine/ItJoinTest.java | 11 +- .../internal/sql/engine/ItSecondaryIndexTest.java | 265 ++- .../sql/engine/exec/AbstractIndexScan.java | 14 +- .../ignite/internal/sql/engine/exec/TreeIndex.java | 3 +- .../sql/engine/exec/exp/ExpressionFactoryImpl.java | 48 +- .../sql/engine/exec/exp/RangeIterable.java | 8 +- .../sql/engine/exec/rel/IndexScanNode.java | 1 - .../internal/sql/engine/rel/AbstractIndexScan.java | 2 +- .../exec/rel/IndexScanNodeExecutionTest.java | 2 +- .../exec/rel/SortedIndexSpoolExecutionTest.java| 5 +- .../sql/engine/planner/AbstractPlannerTest.java| 46 +- .../planner/IndexSearchBoundsPlannerTest.java | 510 + .../internal/sql/engine/planner/PlannerTest.java | 118 - 15 files changed, 849 insertions(+), 191 deletions(-) diff --git a/modules/rest-api/openapi/openapi.yaml b/modules/rest-api/openapi/openapi.yaml index e7b2305b49..10d9f4e941 100644 --- a/modules/rest-api/openapi/openapi.yaml +++ b/modules/rest-api/openapi/openapi.yaml @@ -852,12 +852,15 @@ components: properties: id: type: string + description: Returns unit identifier. versionToConsistentIds: type: object additionalProperties: type: array items: type: string + description: Returns map from existing unit version to list of nodes consistent +ids where unit deployed. description: Unit status. Void: type: object diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItHashSpoolIntegrationTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItHashSpoolIntegrationTest.java index 3101f099f2..cef1e09ad1 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItHashSpoolIntegrationTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItHashSpoolIntegrationTest.java @@ -50,11 +50,11 @@ public class ItHashSpoolIntegrationTest extends ClusterPerClassIntegrationTest { sql("INSERT INTO t0(i1, i2) VALUES (null, 0), (1, null), (null, 2), (3, null), (1, 1)"); sql("INSERT INTO t1(i1, i2) VALUES (null, 0), (null, 1), (2, null), (3, null), (1, 1)"); -String sql = "SELECT /*+ DISABLE_RULE ('MergeJoinConverter', 'NestedLoopJoinConverter', " -+ "'FilterSpoolMergeToSortedIndexSpoolRule')*/ t0.i1, t0.i2, t1.i1, t1.i2 " +String sql = "SELECT t0.i1, t0.i2, t1.i1, t1.i2 " + "FROM t0 JOIN t1 ON t0.i1=t1.i1 AND t0.i2=t1.i2"; assertQuery(sql) +.disableRules("MergeJoinConverter", "NestedLoopJoinConverter", "FilterSpoolMergeToSortedIndexSpoolRule") .returns(1, 1, 1, 1) .check(); } diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItJoinTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItJoinTest.java index eb2f787dd0..817646bbea 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItJoinTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItJoinTest.java @@ -794,17 +794,14 @@ public class ItJoinTest extends ClusterPerClassIntegrationTest { @ParameterizedTest(name = "join algo : {0}, index present: {1}") @MethodSource("joinTypes") @WithSystemProperty(key = "IMPLICIT_PK_ENABLED", value = "true") -public void testIsNotDistinctFrom(JoinType joinType, boolean indexScan) { -if (indexScan) { -// TODO: https://issues.apache.org/jira/browse/IGNITE-18468 Index scan eventually return partial data. -return; -} - +public void testIsNotDistinctFrom(JoinType joinType, boolean indexScan) throws InterruptedException { try { sql("CREATE TABLE t11(i1 INTEGER, i2 INTEGER)"); if (indexScan) { sql("CREATE INDEX t11_idx ON t11(i1)"); +// FIXME: https://issues.apache.org/jira/browse/IGNITE-18203 +waitForIndex("t11_idx");
[ignite] branch IGNITE-17177_inc_snapshots updated (4670f1aaa04 -> 4206dec3834)
This is an automated email from the ASF dual-hosted git repository. timoninmaxim pushed a change to branch IGNITE-17177_inc_snapshots in repository https://gitbox.apache.org/repos/asf/ignite.git from 4670f1aaa04 MINOR: rename restoreIncrementalSnapshot method add 4206dec3834 IGNITE-18967 Fix inc snapshot directories structure (#10580) No new revisions were added by this update. Summary of changes: .../snapshot/IgniteSnapshotManager.java| 39 .../snapshot/IncrementalSnapshotFutureTask.java| 52 +++--- .../snapshot/IncrementalSnapshotMetadata.java | 10 +++-- .../snapshot/SnapshotMetadataVerificationTask.java | 12 ++--- .../snapshot/SnapshotRestoreProcess.java | 42 + .../snapshot/AbstractSnapshotSelfTest.java | 21 + .../snapshot/IncrementalSnapshotTest.java | 10 +++-- .../IncrementalSnapshotCheckBeforeRestoreTest.java | 35 --- .../IncrementalSnapshotRestoreTest.java| 24 +- 9 files changed, 121 insertions(+), 124 deletions(-)
[ignite] branch IGNITE-17177_inc_snapshots updated (196398dff1c -> 4670f1aaa04)
This is an automated email from the ASF dual-hosted git repository. timoninmaxim pushed a change to branch IGNITE-17177_inc_snapshots in repository https://gitbox.apache.org/repos/asf/ignite.git from 196398dff1c IGNITE-17649 Disable incremental snapshots creation after rebalance (#10562) add 4670f1aaa04 MINOR: rename restoreIncrementalSnapshot method No new revisions were added by this update. Summary of changes: .../apache/ignite/util/GridCommandHandlerTest.java | 2 +- .../java/org/apache/ignite/IgniteSnapshot.java | 5 ++- .../snapshot/IgniteSnapshotManager.java| 2 +- .../persistence/snapshot/SnapshotMXBeanImpl.java | 2 +- .../org/apache/ignite/mxbean/SnapshotMXBean.java | 7 ++-- .../snapshot/IgniteSnapshotMXBeanTest.java | 4 +- .../snapshot/IncrementalSnapshotMetricTest.java| 2 +- .../snapshot/IncrementalSnapshotRebalanceTest.java | 4 +- .../IncrementalSnapshotJoiningClientTest.java | 2 +- .../IncrementalSnapshotRestoreTest.java| 44 +++--- .../IncrementalSnapshotWarnAtomicCachesTest.java | 2 +- ...IncrementalSnapshotRestoreWithIndexingTest.java | 2 +- 12 files changed, 40 insertions(+), 38 deletions(-)
[ignite-3] branch main updated: IGNITE-18604 Fix memory leak (#1749)
This is an automated email from the ASF dual-hosted git repository. apkhmv pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/ignite-3.git The following commit(s) were added to refs/heads/main by this push: new 958e3c7f33 IGNITE-18604 Fix memory leak (#1749) 958e3c7f33 is described below commit 958e3c7f33eddbcd59c3a8904b82a5e860fbd37f Author: Mikhail AuthorDate: Mon Mar 6 21:14:05 2023 +0300 IGNITE-18604 Fix memory leak (#1749) - Co-authored-by: Mikhail Pochatkin Co-authored-by: Vadim Pakhnushev <8614891+valep...@users.noreply.github.com> --- .../apache/ignite/internal/rest/RestFactory.java| 11 +++ .../apache/ignite/internal/rest/RestComponent.java | 11 ++- .../internal/rest/RestFactoriesDestroyer.java} | 21 +++-- .../rest/authentication/AuthProviderFactory.java| 7 ++- .../rest/cluster/ClusterManagementRestFactory.java | 10 -- .../rest/configuration/PresentationsFactory.java| 10 -- .../rest/deployment/CodeDeploymentRestFactory.java | 7 ++- .../internal/rest/metrics/MetricRestFactory.java| 7 ++- .../rest/node/NodeManagementRestFactory.java| 10 -- .../org/apache/ignite/internal/app/IgniteImpl.java | 14 -- 10 files changed, 86 insertions(+), 22 deletions(-) diff --git a/modules/rest-api/src/main/java/org/apache/ignite/internal/rest/RestFactory.java b/modules/rest-api/src/main/java/org/apache/ignite/internal/rest/RestFactory.java index 88567cc8b6..6926a2e093 100644 --- a/modules/rest-api/src/main/java/org/apache/ignite/internal/rest/RestFactory.java +++ b/modules/rest-api/src/main/java/org/apache/ignite/internal/rest/RestFactory.java @@ -17,8 +17,19 @@ package org.apache.ignite.internal.rest; +import io.micronaut.runtime.Micronaut; + /** * Factory that produces all beans that is necessary for the controller class. */ public interface RestFactory { +/** + * Destroy method. All resources of the factory implementation must be cleaned and all fields must be set to {@code null}. + * The reason of these requirements is Micronaut design. + * {@link Micronaut#start()} store shutdown hook and capture a pointer + * to the embedded application {@link io.micronaut.http.server.netty.NettyEmbeddedServer} and as a result + * {@link io.micronaut.context.ApplicationContext} will never be collected by the GC. + * All rest factories stored in the application context should be cleaned to prevent memory leak. + */ +void cleanResources(); } diff --git a/modules/rest/src/main/java/org/apache/ignite/internal/rest/RestComponent.java b/modules/rest/src/main/java/org/apache/ignite/internal/rest/RestComponent.java index 9d0e57b1e0..7aa8b5bde0 100644 --- a/modules/rest/src/main/java/org/apache/ignite/internal/rest/RestComponent.java +++ b/modules/rest/src/main/java/org/apache/ignite/internal/rest/RestComponent.java @@ -31,6 +31,7 @@ import java.net.UnknownHostException; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import org.apache.ignite.internal.logger.IgniteLogger; import org.apache.ignite.internal.logger.Loggers; import org.apache.ignite.internal.manager.IgniteComponent; @@ -61,7 +62,7 @@ public class RestComponent implements IgniteComponent { private static final IgniteLogger LOG = Loggers.forClass(RestComponent.class); /** Factories that produce beans needed for REST controllers. */ -private final List restFactories; +private final List> restFactories; private final RestConfiguration restConfiguration; @@ -77,7 +78,7 @@ public class RestComponent implements IgniteComponent { /** * Creates a new instance of REST module. */ -public RestComponent(List restFactories, RestConfiguration restConfiguration) { +public RestComponent(List> restFactories, RestConfiguration restConfiguration) { this.restFactories = restFactories; this.restConfiguration = restConfiguration; } @@ -180,17 +181,17 @@ public class RestComponent implements IgniteComponent { return micronaut .properties(properties) .banner(false) -.mapError(ServerStartupException.class, this::mapServerStartupException) +.mapError(ServerStartupException.class, RestComponent::mapServerStartupException) .mapError(ApplicationStartupException.class, ex -> -1); } private void setFactories(Micronaut micronaut) { for (var factory : restFactories) { -micronaut.singletons(factory); +micronaut.singletons(factory.get()); } } -private int mapServerStartupException(ServerStartupException exception) { +private static int mapServerStartupException(ServerStartupException exception) { if
[ignite-3] branch ignite-18970 updated (4de3c89051 -> 3c3a35bdd7)
This is an automated email from the ASF dual-hosted git repository. sdanilov pushed a change to branch ignite-18970 in repository https://gitbox.apache.org/repos/asf/ignite-3.git from 4de3c89051 IGNITE-18970 Allow WorkDirectoryExtension to preserve work directories of specific tests add 3c3a35bdd7 IGNITE-18970 Allow WorkDirectoryExtension to preserve work directories of specific tests No new revisions were added by this update. Summary of changes: .../testframework/WorkDirectoryExtensionTest.java | 17 + .../internal/testframework/WorkDirectoryExtension.java | 12 2 files changed, 29 insertions(+)
[ignite] branch IGNITE-17177_inc_snapshots updated (b509109eb6e -> 196398dff1c)
This is an automated email from the ASF dual-hosted git repository. timoninmaxim pushed a change to branch IGNITE-17177_inc_snapshots in repository https://gitbox.apache.org/repos/asf/ignite.git from b509109eb6e IGNITE-18069 Check incremental snapshot meta (#10364) add 196398dff1c IGNITE-17649 Disable incremental snapshots creation after rebalance (#10562) No new revisions were added by this update. Summary of changes: .../GridCacheDatabaseSharedManager.java| 2 + .../snapshot/IgniteSnapshotManager.java| 63 .../snapshot/SnapshotRestoreProcess.java | 3 + .../snapshot/IncrementalSnapshotRebalanceTest.java | 173 + .../testsuites/IncrementalSnapshotsTestSuite.java | 2 + 5 files changed, 243 insertions(+) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotRebalanceTest.java
[ignite-3] branch main updated: IGNITE-18730 REST API for Deployment Code API (#1733)
This is an automated email from the ASF dual-hosted git repository. apkhmv pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/ignite-3.git The following commit(s) were added to refs/heads/main by this push: new e104b42b6e IGNITE-18730 REST API for Deployment Code API (#1733) e104b42b6e is described below commit e104b42b6eeb0c44fa01c869bb15944ee741b434 Author: Mikhail AuthorDate: Mon Mar 6 16:19:17 2023 +0300 IGNITE-18730 REST API for Deployment Code API (#1733) --- modules/api/build.gradle | 9 + .../org/apache/ignite/deployment/UnitStatus.java | 4 +- .../ignite/deployment/version/UnitVersion.java | 12 +- .../apache/ignite/deployment/version/Version.java | 5 + .../deployment/version/VersionParseException.java | 17 +- .../testframework}/IntegrationTestBase.java| 136 +-- modules/cli/build.gradle | 5 +- ...onTestBase.java => CliIntegrationTestBase.java} | 167 + .../call/CallInitializedIntegrationTestBase.java | 4 +- ...liCommandTestNotInitializedIntegrationBase.java | 4 +- .../internal/deployunit/DeploymentManagerImpl.java | 1 + modules/rest-api/openapi/openapi.yaml | 224 + .../rest/api/deployment/DeploymentCodeApi.java | 193 +++ .../rest/api/deployment/UnitStatusDto.java | 90 +++ .../ignite/internal/rest/constants/HttpCode.java | 2 + .../ignite/internal/rest/constants/MediaType.java | 4 + .../exception/ClusterNotInitializedException.java | 3 - modules/rest/build.gradle | 5 +- .../DeploymentManagementControllerTest.java| 270 + .../deployment/CodeDeploymentRestFactory.java} | 40 ++- .../deployment/DeploymentManagementController.java | 99 ...DeploymentUnitAlreadyExistExceptionHandler.java | 44 .../DeploymentUnitNotFoundExceptionHandler.java| 45 .../handler/VersionParseExceptionHandler.java | 44 .../org/apache/ignite/internal/app/IgniteImpl.java | 7 +- 25 files changed, 1091 insertions(+), 343 deletions(-) diff --git a/modules/api/build.gradle b/modules/api/build.gradle index 2fcef62ce9..0cca13ad05 100644 --- a/modules/api/build.gradle +++ b/modules/api/build.gradle @@ -18,6 +18,7 @@ apply from: "$rootDir/buildscripts/java-core.gradle" apply from: "$rootDir/buildscripts/publishing.gradle" apply from: "$rootDir/buildscripts/java-junit5.gradle" +apply from: "$rootDir/buildscripts/java-test-fixtures.gradle" dependencies { @@ -30,6 +31,14 @@ dependencies { testImplementation libs.hamcrest.optional testImplementation libs.archunit.core testImplementation libs.archunit.junit5 + +testFixturesAnnotationProcessor project(":ignite-configuration-annotation-processor") +testFixturesAnnotationProcessor libs.micronaut.inject.annotation.processor +testFixturesImplementation project(":ignite-core") +testFixturesImplementation testFixtures(project(":ignite-core")) +testFixturesImplementation project(":ignite-configuration") +testFixturesImplementation libs.hamcrest.core +testFixturesImplementation libs.micronaut.junit5 } description = 'ignite-api' diff --git a/modules/api/src/main/java/org/apache/ignite/deployment/UnitStatus.java b/modules/api/src/main/java/org/apache/ignite/deployment/UnitStatus.java index 771a0fd8db..a9ef4f5a4c 100644 --- a/modules/api/src/main/java/org/apache/ignite/deployment/UnitStatus.java +++ b/modules/api/src/main/java/org/apache/ignite/deployment/UnitStatus.java @@ -18,11 +18,11 @@ package org.apache.ignite.deployment; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeMap; import org.apache.ignite.deployment.version.Version; import org.apache.ignite.internal.tostring.S; @@ -123,7 +123,7 @@ public class UnitStatus { public static class UnitStatusBuilder { private final String id; -private final Map> versionToConsistentIds = new HashMap<>(); +private final Map> versionToConsistentIds = new TreeMap<>(Version::compareTo); /** * Constructor. diff --git a/modules/api/src/main/java/org/apache/ignite/deployment/version/UnitVersion.java b/modules/api/src/main/java/org/apache/ignite/deployment/version/UnitVersion.java index 02b2df683d..08bfd8d9e4 100644 --- a/modules/api/src/main/java/org/apache/ignite/deployment/version/UnitVersion.java +++ b/modules/api/src/main/java/org/apache/ignite/deployment/version/UnitVersion.java @@ -49,16 +49,16 @@ public class UnitVersion implements Version { /** * Parse string representation of version to {@link UnitVersion} if possible. * - * @param s String representation of version. + * @param rawVersion String representation of version. * @return Instance of
[ignite-3] branch main updated: IGNITE-18749 Sql. Planner optimization for JOIN (#1743)
This is an automated email from the ASF dual-hosted git repository. korlov pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/ignite-3.git The following commit(s) were added to refs/heads/main by this push: new 2671f8ae31 IGNITE-18749 Sql. Planner optimization for JOIN (#1743) 2671f8ae31 is described below commit 2671f8ae310e83309c09d96479cbfb5067344b77 Author: korlov42 AuthorDate: Mon Mar 6 14:13:27 2023 +0200 IGNITE-18749 Sql. Planner optimization for JOIN (#1743) --- .../internal/sql/engine/prepare/IgnitePlanner.java | 19 ++-- .../internal/sql/engine/prepare/PlannerHelper.java | 90 -- .../ignite/internal/sql/engine/util/Commons.java | 21 + ...{TpchBenchmark.java => TpchParseBenchmark.java} | 45 +++-- ...pchBenchmark.java => TpchPrepareBenchmark.java} | 22 +++-- .../sql/engine/benchmarks/TpchQueries.java | 2 +- .../sql/engine/framework/TestBuilders.java | 12 +++ .../internal/sql/engine/framework/TestNode.java| 16 .../sql/engine/planner/JoinCommutePlannerTest.java | 102 + 9 files changed, 264 insertions(+), 65 deletions(-) diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgnitePlanner.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgnitePlanner.java index 26424dbc24..4e041770a9 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgnitePlanner.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/IgnitePlanner.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.sql.engine.prepare; +import static org.apache.ignite.internal.sql.engine.util.Commons.shortRuleName; import static org.apache.ignite.lang.ErrorGroups.Sql.QUERY_INVALID_ERR; import java.io.PrintWriter; @@ -482,15 +483,17 @@ public class IgnitePlanner implements Planner, RelOptTable.ViewExpander { } /** - * SetDisabledRules. - * TODO Documentation https://issues.apache.org/jira/browse/IGNITE-15859 + * Sets names of the rules which should be excluded from query optimization pipeline. + * + * @param disabledRuleNames Names of the rules to exclude. The name can be derived from rule by + * {@link Commons#shortRuleName(RelOptRule)}. */ public void setDisabledRules(Set disabledRuleNames) { ctx.rulesFilter(rulesSet -> { List newSet = new ArrayList<>(); for (RelOptRule r : rulesSet) { -if (!disabledRuleNames.contains(shortRuleName(r.toString( { +if (!disabledRuleNames.contains(shortRuleName(r))) { newSet.add(r); } } @@ -499,16 +502,6 @@ public class IgnitePlanner implements Planner, RelOptTable.ViewExpander { }); } -private static String shortRuleName(String ruleDesc) { -int pos = ruleDesc.indexOf('('); - -if (pos == -1) { -return ruleDesc; -} - -return ruleDesc.substring(0, pos); -} - private static class VolcanoPlannerExt extends VolcanoPlanner { protected VolcanoPlannerExt(RelOptCostFactory costFactory, Context externalCtx) { super(costFactory, externalCtx); diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerHelper.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerHelper.java index 8083303d11..8717c21644 100644 --- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerHelper.java +++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/prepare/PlannerHelper.java @@ -17,12 +17,19 @@ package org.apache.ignite.internal.sql.engine.prepare; +import static org.apache.ignite.internal.sql.engine.util.Commons.shortRuleName; + import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelHomogeneousShuttle; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.logical.LogicalCorrelate; +import org.apache.calcite.rel.rules.CoreRules; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlNode; @@ -36,10 +43,17 @@ import org.apache.ignite.internal.sql.engine.trait.IgniteDistributions; import org.apache.ignite.internal.sql.engine.util.HintUtils; /** - * PlannerHelper. - * TODO Documentation https://issues.apache.org/jira/browse/IGNITE-15859 + * Utility class that encapsulates the query optimization pipeline. */ -public class PlannerHelper { +public final class PlannerHelper { +/** + * Maximum number of tables in join supported for join order
[ignite-3] branch ignite-18208 updated (28229ed7d0 -> d24606a382)
This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a change to branch ignite-18208 in repository https://gitbox.apache.org/repos/asf/ignite-3.git from 28229ed7d0 WIP. Rework after review 2/2. add d24606a382 code cleanup. No new revisions were added by this update. Summary of changes: .../planner/AbstractAggregatePlannerTest.java | 202 + 1 file changed, 1 insertion(+), 201 deletions(-)
[ignite-3] branch main updated: IGNITE-18850 Add ssl configuration validation (#1738)
This is an automated email from the ASF dual-hosted git repository. apkhmv pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/ignite-3.git The following commit(s) were added to refs/heads/main by this push: new 8875dc414b IGNITE-18850 Add ssl configuration validation (#1738) 8875dc414b is described below commit 8875dc414b00fe6e8f42a05eece63ac5eabad5b0 Author: Vadim Pakhnushev <8614891+valep...@users.noreply.github.com> AuthorDate: Mon Mar 6 14:20:37 2023 +0300 IGNITE-18850 Add ssl configuration validation (#1738) --- .../validation/PowerOfTwoValidatorTest.java| 4 +- .../validation/RangeValidatorTest.java | 22 ++-- .../validation/TestValidationUtil.java | 37 -- .../AbstractSslConfigurationSchema.java| 2 - .../KeyStoreConfigurationValidatorImpl.java| 57 - .../configuration/NetworkConfigurationSchema.java | 1 + ...lidator.java => SslConfigurationValidator.java} | 4 +- .../SslConfigurationValidatorImpl.java | 76 +++ .../KeyStoreConfigurationValidatorImplTest.java| 77 .../SslConfigurationValidatorImplTest.java | 127 +++ .../network/configuration/StubSslView.java}| 44 +-- .../apache/ignite/internal/rest/RestComponent.java | 72 ++- .../configuration/RestConfigurationModule.java | 4 +- .../ignite/internal/rest/RestComponentTest.java| 139 - .../KnownDataStorageValidatorTest.java | 2 +- .../configuration/TableValidatorImplTest.java | 2 +- .../ExistingDataStorageValidatorTest.java | 6 +- 17 files changed, 303 insertions(+), 373 deletions(-) diff --git a/modules/configuration/src/test/java/org/apache/ignite/internal/configuration/validation/PowerOfTwoValidatorTest.java b/modules/configuration/src/test/java/org/apache/ignite/internal/configuration/validation/PowerOfTwoValidatorTest.java index df7f3a8934..035c795153 100644 --- a/modules/configuration/src/test/java/org/apache/ignite/internal/configuration/validation/PowerOfTwoValidatorTest.java +++ b/modules/configuration/src/test/java/org/apache/ignite/internal/configuration/validation/PowerOfTwoValidatorTest.java @@ -34,8 +34,8 @@ public class PowerOfTwoValidatorTest { PowerOfTwoValidator validator = new PowerOfTwoValidator(); -validate(validator, powerOfTwo, mockValidationContext(null, 1), null); -validate(validator, powerOfTwo, mockValidationContext(null, 1L), null); +validate(validator, powerOfTwo, mockValidationContext(null, 1)); +validate(validator, powerOfTwo, mockValidationContext(null, 1L)); } @Test diff --git a/modules/configuration/src/test/java/org/apache/ignite/internal/configuration/validation/RangeValidatorTest.java b/modules/configuration/src/test/java/org/apache/ignite/internal/configuration/validation/RangeValidatorTest.java index e27a9a5123..c7a0067ab3 100644 --- a/modules/configuration/src/test/java/org/apache/ignite/internal/configuration/validation/RangeValidatorTest.java +++ b/modules/configuration/src/test/java/org/apache/ignite/internal/configuration/validation/RangeValidatorTest.java @@ -36,23 +36,23 @@ public class RangeValidatorTest { RangeValidator validator = new RangeValidator(); -validate(validator, range0, mockValidationContext(null, 0), null); -validate(validator, range0, mockValidationContext(null, 50), null); -validate(validator, range0, mockValidationContext(null, 100), null); +validate(validator, range0, mockValidationContext(null, 0)); +validate(validator, range0, mockValidationContext(null, 50)); +validate(validator, range0, mockValidationContext(null, 100)); Range range1 = createRange(0L, null); -validate(validator, range1, mockValidationContext(null, 0), null); -validate(validator, range1, mockValidationContext(null, 50), null); -validate(validator, range1, mockValidationContext(null, 100), null); -validate(validator, range1, mockValidationContext(null, Long.MAX_VALUE), null); +validate(validator, range1, mockValidationContext(null, 0)); +validate(validator, range1, mockValidationContext(null, 50)); +validate(validator, range1, mockValidationContext(null, 100)); +validate(validator, range1, mockValidationContext(null, Long.MAX_VALUE)); Range range2 = createRange(null, 100L); -validate(validator, range2, mockValidationContext(null, 0), null); -validate(validator, range2, mockValidationContext(null, 50), null); -validate(validator, range2, mockValidationContext(null, 100), null); -validate(validator, range2, mockValidationContext(null, Long.MIN_VALUE), null); +validate(validator, range2, mockValidationContext(null, 0)); +validate(validator, range2, mockValidationContext(null, 50)); +
[ignite-3] branch ignite-18208 updated (ec4402bfb9 -> 28229ed7d0)
This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a change to branch ignite-18208 in repository https://gitbox.apache.org/repos/asf/ignite-3.git omit ec4402bfb9 WIP. Rework after review 2/2. add 28229ed7d0 WIP. Rework after review 2/2. This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (ec4402bfb9) \ N -- N -- N refs/heads/ignite-18208 (28229ed7d0) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .../planner/AbstractAggregatePlannerTest.java | 124 ++--- .../sql/engine/planner/AggregatePlannerTest.java | 4 +- .../planner/ColocatedHashAggregatePlannerTest.java | 4 +- .../planner/ColocatedSortAggregatePlannerTest.java | 6 +- .../planner/MapReduceHashAggregatePlannerTest.java | 4 +- .../planner/MapReduceSortAggregatePlannerTest.java | 4 +- 6 files changed, 73 insertions(+), 73 deletions(-)
[ignite-3] branch ignite-18208 updated (10f3c4900a -> ec4402bfb9)
This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a change to branch ignite-18208 in repository https://gitbox.apache.org/repos/asf/ignite-3.git from 10f3c4900a WIP. Rework after review 1/2. add ec4402bfb9 WIP. Rework after review 2/2. No new revisions were added by this update. Summary of changes: .../sql/engine/planner/AggregatePlannerTest.java | 680 +- .../planner/ColocatedHashAggregatePlannerTest.java | 2 - .../planner/ColocatedSortAggregatePlannerTest.java | 3 - .../planner/MapReduceHashAggregatePlannerTest.java | 695 +-- .../planner/MapReduceSortAggregatePlannerTest.java | 763 +++-- 5 files changed, 1074 insertions(+), 1069 deletions(-)
[ignite-3] branch ignite-18970 updated (44a812022a -> 4de3c89051)
This is an automated email from the ASF dual-hosted git repository. sdanilov pushed a change to branch ignite-18970 in repository https://gitbox.apache.org/repos/asf/ignite-3.git from 44a812022a IGNITE-18970 Allow WorkDirectoryExtension to preserve work directories of specific tests add 4de3c89051 IGNITE-18970 Allow WorkDirectoryExtension to preserve work directories of specific tests No new revisions were added by this update. Summary of changes: .../ignite/internal/testframework/WorkDirectoryExtensionTest.java | 7 ++- 1 file changed, 2 insertions(+), 5 deletions(-)
[ignite-3] branch ignite-18970 updated (c0e5d087cd -> 44a812022a)
This is an automated email from the ASF dual-hosted git repository. sdanilov pushed a change to branch ignite-18970 in repository https://gitbox.apache.org/repos/asf/ignite-3.git from c0e5d087cd IGNITE-18970 Allow WorkDirectoryExtension to preserve work directories of specific tests add 44a812022a IGNITE-18970 Allow WorkDirectoryExtension to preserve work directories of specific tests No new revisions were added by this update. Summary of changes: .../testframework/WorkDirectoryExtensionTest.java | 46 +- 1 file changed, 45 insertions(+), 1 deletion(-)
[ignite-3] 01/01: IGNITE-18970 Allow WorkDirectoryExtension to preserve work directories of specific tests
This is an automated email from the ASF dual-hosted git repository. sdanilov pushed a commit to branch ignite-18970 in repository https://gitbox.apache.org/repos/asf/ignite-3.git commit c0e5d087cda1d7ee2c7f303d3c5eb54e6f629c1d Author: Semyon Danilov AuthorDate: Mon Mar 6 14:46:22 2023 +0400 IGNITE-18970 Allow WorkDirectoryExtension to preserve work directories of specific tests --- .../testframework/WorkDirectoryExtensionTest.java | 77 - .../testframework/WorkDirectoryExtension.java | 99 ++ 2 files changed, 158 insertions(+), 18 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/testframework/WorkDirectoryExtensionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/testframework/WorkDirectoryExtensionTest.java index d56f78ef1e..2553d00e13 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/testframework/WorkDirectoryExtensionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/testframework/WorkDirectoryExtensionTest.java @@ -31,8 +31,10 @@ import static org.junit.platform.testkit.engine.TestExecutionResultConditions.me import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.HashSet; import java.util.Set; +import org.apache.ignite.internal.util.IgniteUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -204,16 +206,33 @@ class WorkDirectoryExtensionTest { private static Path file2; +private static Path file3; + +private static final String TMP_PATH; + +static { +try { +TMP_PATH = Files.createTempDirectory("testdir").toString(); +} catch (IOException e) { +throw new RuntimeException(e); +} +} + @AfterAll -static void verify() throws IOException { +static void verify() { assertTrue(Files.exists(file1)); assertFalse(Files.exists(file2)); +assertFalse(Files.exists(file3)); +assertTrue(Files.exists(Paths.get(TMP_PATH))); + +IgniteUtils.deleteIfExists(file1.getParent()); +IgniteUtils.deleteIfExists(Paths.get(TMP_PATH)); -Files.delete(file1); +System.clearProperty(WorkDirectoryExtension.ARTIFACT_DIR_PROPERTY); } @SuppressWarnings("AssignmentToStaticFieldFromInstanceMethod") -@WithSystemProperty(key = WorkDirectoryExtension.KEEP_WORK_DIR_PROPERTY, value = "true") +@WithSystemProperty(key = WorkDirectoryExtension.KEEP_WORK_DIR_PROPERTY, value = "SystemPropertiesTest.test1") @Test void test1(@WorkDirectory Path workDir) throws IOException { file1 = Files.createFile(workDir.resolve("foo")); @@ -224,6 +243,50 @@ class WorkDirectoryExtensionTest { void test2(@WorkDirectory Path workDir) throws IOException { file2 = Files.createFile(workDir.resolve("foo")); } + +@SuppressWarnings("AssignmentToStaticFieldFromInstanceMethod") +@WithSystemProperty(key = WorkDirectoryExtension.KEEP_WORK_DIR_PROPERTY, value = "SystemPropertiesTest.test3") +@Test +void test3(@WorkDirectory Path workDir) throws IOException { +file3 = Files.createFile(workDir.resolve("foo")); + +System.setProperty(WorkDirectoryExtension.ARTIFACT_DIR_PROPERTY, TMP_PATH); +} +} + +/** + * Test class for the {@link #testSystemPropertyWithStaticWorkDir()} test. + */ +@ExtendWith(SystemPropertiesExtension.class) +@ExtendWith(WorkDirectoryExtension.class) +@WithSystemProperty(key = WorkDirectoryExtension.KEEP_WORK_DIR_PROPERTY, value = "SystemPropertiesTestWithStaticWorkDir") +static class SystemPropertiesTestWithStaticWorkDir { +private static Path file1; + +private static Path file2; + +@WorkDirectory +static Path workDir; + +@AfterAll +static void verify() { +assertTrue(Files.exists(file1)); +assertTrue(Files.exists(file2)); + +IgniteUtils.deleteIfExists(file1.getParent()); +} + +@SuppressWarnings("AssignmentToStaticFieldFromInstanceMethod") +@Test +void test1() throws IOException { +file1 = Files.createFile(workDir.resolve("foo")); +} + +@SuppressWarnings("AssignmentToStaticFieldFromInstanceMethod") +@Test +void test2() throws IOException { +file2 = Files.createFile(workDir.resolve("bar")); +} } /** @@ -234,6 +297,14 @@ class WorkDirectoryExtensionTest { assertExecutesSuccessfully(SystemPropertiesTest.class); } +/** + * Tests that a static work directory can be preserved when a special system property is set. + */ +@Test +
[ignite-3] branch ignite-18970 created (now c0e5d087cd)
This is an automated email from the ASF dual-hosted git repository. sdanilov pushed a change to branch ignite-18970 in repository https://gitbox.apache.org/repos/asf/ignite-3.git at c0e5d087cd IGNITE-18970 Allow WorkDirectoryExtension to preserve work directories of specific tests This branch includes the following new commits: new c0e5d087cd IGNITE-18970 Allow WorkDirectoryExtension to preserve work directories of specific tests The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[ignite] branch master updated (d5af66c5658 -> 248d9e562ce)
This is an automated email from the ASF dual-hosted git repository. mpetrov pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git from d5af66c5658 IGNITE-18819 Added cluster state change permission (#10558) add 248d9e562ce IGNITE-18923 Added authorization of user management operations. (#10567) No new revisions were added by this update. Summary of changes: .../common/AbstractEventSecurityContextTest.java | 23 +- ...CacheCreateDestroyEventSecurityContextTest.java | 6 +- .../common/CacheEventSecurityContextTest.java | 6 +- .../ComputeTaskRemoteSecurityContextTest.java | 34 .../apache/ignite/jdbc/JdbcAuthorizationTest.java | 94 ++ .../JdbcThinConnectionAdditionalSecurityTest.java | 26 -- .../security/IgniteSecurityProcessor.java | 7 ++ .../ignite/plugin/security/SecurityPermission.java | 3 + .../processors/security/AbstractSecurityTest.java | 16 +++- .../processors/security/InvalidServerTest.java | 3 +- .../impl/TestCertificateSecurityProcessor.java | 2 +- .../security/impl/TestSecurityContext.java | 1 + .../processors/security/impl/TestSecurityData.java | 81 +++ .../security/impl/TestSecurityProcessor.java | 77 +- .../ignite/platform/PlatformStartIgniteUtils.java | 19 +++-- .../systemview/SystemViewSecurityTest.java | 10 ++- .../http/jetty/RestProcessorAuthorizationTest.java | 5 ++ 17 files changed, 262 insertions(+), 151 deletions(-)
[ignite-3] branch main updated: IGNITE-18938 SSL integration tests fail on Windows (#1742)
This is an automated email from the ASF dual-hosted git repository. apkhmv pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/ignite-3.git The following commit(s) were added to refs/heads/main by this push: new 10a58e67a2 IGNITE-18938 SSL integration tests fail on Windows (#1742) 10a58e67a2 is described below commit 10a58e67a2551f8421d9603ccc97d0f6581ec1b2 Author: Vadim Pakhnushev <8614891+valep...@users.noreply.github.com> AuthorDate: Mon Mar 6 11:56:49 2023 +0300 IGNITE-18938 SSL integration tests fail on Windows (#1742) --- .../org/apache/ignite/internal/rest/RestNode.java | 25 -- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/rest/RestNode.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/rest/RestNode.java index 844ab29ce7..1a955c1154 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/rest/RestNode.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/rest/RestNode.java @@ -17,7 +17,10 @@ package org.apache.ignite.internal.rest; +import java.net.URISyntaxException; +import java.net.URL; import java.nio.file.Path; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import org.apache.ignite.Ignite; import org.apache.ignite.IgnitionManager; @@ -112,9 +115,6 @@ public class RestNode { } private String bootstrapCfg() { -String keyStoreAbsolutPath = ItRestSslTest.class.getClassLoader().getResource(keyStorePath).getPath(); -String trustStoreAbsolutPath = ItRestSslTest.class.getClassLoader().getResource(trustStorePath).getPath(); - return "{\n" + " network: {\n" + "port: " + networkPort + ",\n" @@ -130,16 +130,27 @@ public class RestNode { + " clientAuth: " + (sslClientAuthEnabled ? "require" : "none") + ",\n" + " port: " + httpsPort + ",\n" + " keyStore: {\n" -+ "path: " + keyStoreAbsolutPath + ",\n" ++ "path: \"" + getResourcePath(keyStorePath) + "\",\n" + "password: " + keyStorePassword + "\n" + " }, \n" + " trustStore: {\n" -+ "type: JKS, " -+ "path: " + trustStoreAbsolutPath + ",\n" ++ "type: JKS,\n" ++ "path: \"" + getResourcePath(trustStorePath) + "\",\n" + "password: " + trustStorePassword + "\n" + " }\n" + "}\n" -+ " }" ++ " }\n" + "}"; } + +private static String getResourcePath(String resource) { +try { +URL url = ItRestSslTest.class.getClassLoader().getResource(resource); +Objects.requireNonNull(url, "Resource " + resource + " not found."); +Path path = Path.of(url.toURI()); // Properly extract file system path from the "file:" URL +return path.toString().replace("\\", ""); // Escape backslashes for the config parser +} catch (URISyntaxException e) { +throw new RuntimeException(e); // Shouldn't happen since URL is obtained from the class loader +} +} }
[ignite-3] branch main updated: IGNITE-18926 Open API spec validation fails (#1744)
This is an automated email from the ASF dual-hosted git repository. apkhmv pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/ignite-3.git The following commit(s) were added to refs/heads/main by this push: new 2013940f1e IGNITE-18926 Open API spec validation fails (#1744) 2013940f1e is described below commit 2013940f1e8fc89553648ab09b156613c6eb3483 Author: Vadim Pakhnushev <8614891+valep...@users.noreply.github.com> AuthorDate: Mon Mar 6 11:47:31 2023 +0300 IGNITE-18926 Open API spec validation fails (#1744) --- docs/_docs/rest/rest-api.adoc | 6 ++-- modules/cli/build.gradle | 2 +- modules/rest-api/build.gradle | 10 ++- .../{rest => rest-api}/openapi/openapi.properties | 0 modules/{rest => rest-api}/openapi/openapi.yaml| 29 -- .../ignite/internal/rest/api/Definition.java | 34 ++ .../internal/rest/api/cluster/ClusterNodeDto.java | 5 .../internal/rest/api/cluster/InitCommand.java | 1 + .../api/configuration/ClusterConfigurationApi.java | 5 +++- .../api/configuration/NodeConfigurationApi.java| 5 +++- modules/rest/build.gradle | 13 ++--- .../apache/ignite/internal/rest/RestComponent.java | 24 --- 12 files changed, 85 insertions(+), 49 deletions(-) diff --git a/docs/_docs/rest/rest-api.adoc b/docs/_docs/rest/rest-api.adoc index bd690c70a6..5901089958 100644 --- a/docs/_docs/rest/rest-api.adoc +++ b/docs/_docs/rest/rest-api.adoc @@ -15,7 +15,7 @@ = Ignite REST API -The Apache Ignite 3 clusters provide an link:https://www.openapis.org/[OpenAPI] specification that can be used to work with Ignite 3 by standard REST methods. The link:https://github.com/apache/ignite-3/tree/main/modules/rest/openapi/openapi.yaml[openapi.yaml,window=_blank] specification file for the version is included with its release. +The Apache Ignite 3 clusters provide an link:https://www.openapis.org/[OpenAPI] specification that can be used to work with Ignite 3 by standard REST methods. The link:https://github.com/apache/ignite-3/tree/main/modules/rest-api/openapi/openapi.yaml[openapi.yaml,window=_blank] specification file for the version is included with its release. We recommend that you generate client code in your project language by using an link:https://github.com/OpenAPITools/openapi-generator[OpenAPI code generator,window=_blank]. Below is the example of how you can do this for a Java project. @@ -35,7 +35,7 @@ We recommend that you generate client code in your project language by using an generate - https://github.com/apache/ignite-3/tree/main/modules/rest/openapi/openapi.yaml + https://github.com/apache/ignite-3/tree/main/modules/rest-api/openapi/openapi.yaml java org.apache.ignite.rest.client.api org.apache.ignite.rest.client.invoker @@ -65,4 +65,4 @@ client.setBasePath("http://localhost:10300;); // Get cluster configuration. clusterConfigurationApi = new ClusterConfigurationApi(client); String configuration = clusterConfigurationApi.getClusterConfiguration(); - \ No newline at end of file + diff --git a/modules/cli/build.gradle b/modules/cli/build.gradle index aa408e17f9..59d5ee5d85 100644 --- a/modules/cli/build.gradle +++ b/modules/cli/build.gradle @@ -111,7 +111,7 @@ dependencies { ext.generatedClientDir = layout.buildDirectory.dir("swagger/client") task copyOpenapiDefinition(type: Copy) { -from("$rootDir/modules/rest/openapi") +from("$rootDir/modules/rest-api/openapi") into("$buildDir/customOpenapiDefinition/") } diff --git a/modules/rest-api/build.gradle b/modules/rest-api/build.gradle index 2cb8b518aa..989ffd9b7a 100644 --- a/modules/rest-api/build.gradle +++ b/modules/rest-api/build.gradle @@ -19,8 +19,12 @@ apply from: "$rootDir/buildscripts/java-core.gradle" apply from: "$rootDir/buildscripts/publishing.gradle" apply from: "$rootDir/buildscripts/java-junit5.gradle" +description = 'ignite-rest-api' + dependencies { annotationProcessor libs.micronaut.inject.annotation.processor +annotationProcessor libs.micronaut.openapi + implementation project(':ignite-api') implementation project(':ignite-core') implementation project(':ignite-configuration-api') @@ -34,9 +38,13 @@ dependencies { implementation libs.micronaut.http.server.core implementation libs.micronaut.security implementation libs.micronaut.security.annotations + testImplementation libs.junit5.api testImplementation libs.mockito.core testImplementation libs.junit5.params } -description = 'ignite-rest-api' +compileJava { +options.fork = true +options.forkOptions.jvmArgs += '-Dmicronaut.openapi.config.file=openapi/openapi.properties' +} diff --git a/modules/rest/openapi/openapi.properties b/modules/rest-api/openapi/openapi.properties similarity