This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a change to branch ignite-19775 in repository https://gitbox.apache.org/repos/asf/ignite-3.git
from 963270f21f Styles add 8a87e3d769 IGNITE-19755 Move common test classes to testFixtures for runner module (#2205) add 60acfd0922 IGNITE-19626 .NET: Propagate compute deployment units (#2248) add 6e225a7738 IGNITE-19625 C++ Compute API supports deployment units (#2257) add ff7407fd9a IGNITE-19736 Do not cancel tasks in DistributionZoneManager#executor if they were created by immediate scaleUp/scaleDown events. Avoid concurrent executing several tasks for the same zone. (#2201) add b71a2674cc IGNITE-19689 Fix ItClusterManagerTest.testClusterConfigurationIsRemovedFromClusterStateAfterUpdating (#2250) add 28b4ac4019 IGNITE-19777 Perform local metastorage recovery (#2237) add ef3b4c320d IGNITE-19694 Introduce deployment unit operation ID (#2238) add 5eccdcd8b1 IGNITE-19828 Optimize RAFT log encoder/decoder data format (#2249) add 81fe252ee7 IGNITE-19610 .NET: Add data streamer metrics (#2262) add 5d3a9ad69e IGNITE-19708 Check refcounter of unit before undeploy (#2219) add cae90410a3 IGNITE-19724 Remove redundant joins (#2252) add b5ad99f3b6 Merge branch 'main' into ignite-19775 No new revisions were added by this update. Summary of changes: .../org/apache/ignite/compute/IgniteCompute.java | 8 +- .../apache/ignite/client/ClientComputeTest.java | 1 + .../cluster/management/ItClusterManagerTest.java | 3 +- .../internal/deployunit/DefaultNodeCallback.java | 9 +- .../internal/deployunit/DeploymentManagerImpl.java | 84 ++++-- .../deployunit/DeploymentUnitAccessor.java | 8 +- .../deployunit/DeploymentUnitAccessorImpl.java | 39 ++- .../deployunit/DeploymentUnitAcquiredWaiter.java | 103 +++++++ .../internal/deployunit/IgniteDeployment.java | 2 +- .../ignite/internal/deployunit/UnitStatus.java | 26 +- .../metastore/DeploymentUnitFailover.java | 49 +++- .../deployunit/metastore/DeploymentUnitStore.java | 25 +- .../metastore/DeploymentUnitStoreImpl.java | 19 +- .../metastore/status/SerializeUtils.java | 5 + .../metastore/status/UnitClusterStatus.java | 41 ++- .../metastore/status/UnitNodeStatus.java | 36 ++- .../deployment/UnitStatusesSerializerTest.java | 22 +- .../metastore/DeploymentUnitStoreImplTest.java | 71 +++-- .../deployunit/DeploymentManagerImplTest.java | 10 +- .../deployunit/DeploymentUnitAccessorImplTest.java | 53 ++++ .../DeploymentUnitAcquiredWaiterTest.java | 155 ++++++++++ .../compute/ClassLoaderExceptionsMapper.java | 8 +- .../internal/compute/ComputeComponentImpl.java | 17 +- modules/compute/src/test/README.md | 6 +- .../compute/ClassLoaderExceptionsMapperTest.java | 4 +- ...OT-src.zip => ignite-jobs-1.0-SNAPSHOT-src.zip} | Bin .../unit1/1.0.0/ignite-ut-job1-1.0-SNAPSHOT.jar | Bin 0 -> 1749 bytes .../units/unit1/1.0.0/unit1-1.0-SNAPSHOT.jar | Bin 1749 -> 0 bytes .../unit1/2.0.0/ignite-ut-job2-1.0-SNAPSHOT.jar | Bin 0 -> 1724 bytes .../units/unit1/2.0.0/unit2-1.0-SNAPSHOT.jar | Bin 1724 -> 0 bytes .../unit1/3.0.1/ignite-ut-job1-1.0-SNAPSHOT.jar | Bin 0 -> 1749 bytes .../unit1/3.0.1/ignite-ut-job2-1.0-SNAPSHOT.jar | Bin 0 -> 1724 bytes .../units/unit1/3.0.1/unit1-1.0-SNAPSHOT.jar | Bin 1749 -> 0 bytes .../units/unit1/3.0.1/unit2-1.0-SNAPSHOT.jar | Bin 1724 -> 0 bytes .../unit1/3.0.2/ignite-ut-job1-1.0-SNAPSHOT.jar | Bin 0 -> 1749 bytes .../3.0.2/subdir/ignite-ut-job2-1.0-SNAPSHOT.jar | Bin 0 -> 1724 bytes .../unit1/3.0.2/subdir/unit2-1.0-SNAPSHOT.jar | Bin 1724 -> 0 bytes .../units/unit1/3.0.2/unit1-1.0-SNAPSHOT.jar | Bin 1749 -> 0 bytes .../unit2/1.0.0/ignite-ut-job1-1.0-SNAPSHOT.jar | Bin 0 -> 1749 bytes .../units/unit2/1.0.0/unit1-1.0-SNAPSHOT.jar | Bin 1749 -> 0 bytes .../unit2/2.0.0/ignite-ut-job2-1.0-SNAPSHOT.jar | Bin 0 -> 1724 bytes .../units/unit2/2.0.0/unit2-1.0-SNAPSHOT.jar | Bin 1724 -> 0 bytes .../distributionzones/DistributionZoneManager.java | 62 ++-- .../distributionzones/DistributionZonesUtil.java | 22 ++ .../DistributionZoneManagerScaleUpTest.java | 312 --------------------- .../DistributionZonesSchedulersTest.java | 275 ++++++++++++++++-- .../internal/metastorage/MetaStorageManager.java | 6 + ...MetaStorageSafeTimePropagationAbstractTest.java | 2 +- ...Command.java => GetCurrentRevisionCommand.java} | 13 +- .../command/MetastorageCommandsMessageGroup.java | 8 +- .../metastorage/impl/MetaStorageManagerImpl.java | 87 +++++- .../metastorage/impl/MetaStorageService.java | 5 + .../metastorage/impl/MetaStorageServiceImpl.java | 8 + .../metastorage/server/KeyValueStorage.java | 12 +- .../server/persistence/RocksDbKeyValueStorage.java | 63 +++-- .../server/raft/MetaStorageListener.java | 5 + .../MetaStorageDeployWatchesCorrectnessTest.java | 7 +- .../impl/MetaStorageManagerRecoveryTest.java | 176 ++++++++++++ .../server/BasicOperationsKeyValueStorageTest.java | 6 +- .../server/RocksDbKeyValueStorageTest.java | 74 ----- .../server/SimpleInMemoryKeyValueStorage.java | 43 ++- .../stream/DirectByteBufferStreamImplV1.java | 4 +- .../MultiActorPlacementDriverTest.java | 10 +- modules/platforms/cpp/ignite/client/CMakeLists.txt | 1 + .../cpp/ignite/client/compute/compute.cpp | 18 +- .../platforms/cpp/ignite/client/compute/compute.h | 42 ++- .../cpp/ignite/client/compute/deployment_unit.h | 81 ++++++ .../ignite/client/detail/compute/compute_impl.cpp | 44 ++- .../ignite/client/detail/compute/compute_impl.h | 9 +- modules/platforms/cpp/ignite/protocol/writer.h | 9 + .../tests/client-test/basic_authenticator_test.cpp | 2 +- .../cpp/tests/client-test/compute_test.cpp | 159 ++++++++++- .../Apache.Ignite.Tests/BasicAuthenticatorTests.cs | 4 +- .../Compute/ComputeClusterAwarenessTests.cs | 20 +- .../Apache.Ignite.Tests/Compute/ComputeTests.cs | 120 ++++++-- .../dotnet/Apache.Ignite.Tests/FakeServer.cs | 58 +++- .../Linq/LinqSqlGenerationTests.cs | 2 +- .../dotnet/Apache.Ignite.Tests/MetricsTests.cs | 87 ++++++ .../Apache.Ignite.Tests/PartitionAwarenessTests.cs | 11 +- .../Proto/ColocationHashTests.cs | 10 +- .../platforms/dotnet/Apache.Ignite.sln.DotSettings | 1 + .../ResponseFlags.cs => Compute/DeploymentUnit.cs} | 19 +- .../dotnet/Apache.Ignite/Compute/ICompute.cs | 121 ++++---- .../Apache.Ignite/Internal/Compute/Compute.cs | 93 +++++- .../dotnet/Apache.Ignite/Internal/Metrics.cs | 59 ++++ .../Apache.Ignite/Internal/Table/DataStreamer.cs | 24 +- .../internal/raft/server/impl/JraftServerImpl.java | 2 +- .../internal/raft/util/OptimizedMarshaller.java | 4 +- .../raft/util/ThreadLocalOptimizedMarshaller.java | 3 +- .../entity/codec/v1/LogEntryV1CodecFactory.java | 22 +- .../raft/jraft/entity/codec/v1/V1Decoder.java | 196 +++++++------ .../raft/jraft/entity/codec/v1/V1Encoder.java | 239 ++++++++-------- .../ignite/raft/jraft/storage/io/MessageFile.java | 3 +- .../snapshot/local/LocalSnapshotMetaTable.java | 2 +- .../ignite/raft/jraft/util/AsciiStringUtil.java | 16 +- .../ignite/raft/jraft/util/JDKMarshaller.java | 5 +- .../apache/ignite/raft/jraft/util/Marshaller.java | 4 +- .../jraft/entity/codec/LogEntryCodecPerfTest.java | 3 +- .../ignite/internal/replicator/ReplicaManager.java | 3 +- modules/runner/build.gradle | 10 + .../internal/compute/ItComputeTestStandalone.java | 49 +++- .../internal/deployment/ItDeploymentUnitTest.java | 28 ++ .../ignite/internal/rebalance/ItRebalanceTest.java | 6 +- .../runner/app/ItIgniteNodeRestartTest.java | 87 +++++- .../runner/app/client/ItThinClientComputeTest.java | 35 +++ .../apache/ignite/internal/start/ItStartTest.java | 3 +- .../units/ignite-it-jobs-1.0-SNAPSHOT.jar | Bin 0 -> 5528 bytes ...OT-src.zip => ignite-jobs-1.0-SNAPSHOT-src.zip} | Bin .../resources/units/ignite-jobs-1.0-SNAPSHOT.jar | Bin 4773 -> 0 bytes .../units/ignite-ut-job1-1.0-SNAPSHOT.jar | Bin 0 -> 1749 bytes .../units/ignite-ut-job2-1.0-SNAPSHOT.jar | Bin 0 -> 1724 bytes .../resources/units/unit1-1.0-SNAPSHOT.jar | Bin 1749 -> 0 bytes .../resources/units/unit2-1.0-SNAPSHOT.jar | Bin 1724 -> 0 bytes .../org/apache/ignite/internal/app/IgniteImpl.java | 15 +- .../java/org/apache/ignite/internal/Cluster.java | 0 .../internal/sql/engine/util/CursorUtils.java | 0 .../internal/sql/engine/util/MetadataMatcher.java | 0 .../sql/engine/util/NativeTypeWrapper.java | 0 .../internal/sql/engine/util/QueryChecker.java | 0 .../sql/engine/util/TestQueryProcessor.java | 0 .../ignite/internal/sql/engine/util/VarBinary.java | 0 .../ignite/distributed/ReplicaUnavailableTest.java | 3 +- .../internal/table/distributed/TableManager.java | 3 - 123 files changed, 2701 insertions(+), 1038 deletions(-) create mode 100644 modules/code-deployment/src/main/java/org/apache/ignite/internal/deployunit/DeploymentUnitAcquiredWaiter.java create mode 100644 modules/code-deployment/src/test/java/org/apache/ignite/internal/deployunit/DeploymentUnitAccessorImplTest.java create mode 100644 modules/code-deployment/src/test/java/org/apache/ignite/internal/deployunit/DeploymentUnitAcquiredWaiterTest.java rename modules/compute/src/test/resources/units/{apache-ignite-1.0-SNAPSHOT-src.zip => ignite-jobs-1.0-SNAPSHOT-src.zip} (100%) create mode 100644 modules/compute/src/test/resources/units/unit1/1.0.0/ignite-ut-job1-1.0-SNAPSHOT.jar delete mode 100644 modules/compute/src/test/resources/units/unit1/1.0.0/unit1-1.0-SNAPSHOT.jar create mode 100644 modules/compute/src/test/resources/units/unit1/2.0.0/ignite-ut-job2-1.0-SNAPSHOT.jar delete mode 100644 modules/compute/src/test/resources/units/unit1/2.0.0/unit2-1.0-SNAPSHOT.jar create mode 100644 modules/compute/src/test/resources/units/unit1/3.0.1/ignite-ut-job1-1.0-SNAPSHOT.jar create mode 100644 modules/compute/src/test/resources/units/unit1/3.0.1/ignite-ut-job2-1.0-SNAPSHOT.jar delete mode 100644 modules/compute/src/test/resources/units/unit1/3.0.1/unit1-1.0-SNAPSHOT.jar delete mode 100644 modules/compute/src/test/resources/units/unit1/3.0.1/unit2-1.0-SNAPSHOT.jar create mode 100644 modules/compute/src/test/resources/units/unit1/3.0.2/ignite-ut-job1-1.0-SNAPSHOT.jar create mode 100644 modules/compute/src/test/resources/units/unit1/3.0.2/subdir/ignite-ut-job2-1.0-SNAPSHOT.jar delete mode 100644 modules/compute/src/test/resources/units/unit1/3.0.2/subdir/unit2-1.0-SNAPSHOT.jar delete mode 100644 modules/compute/src/test/resources/units/unit1/3.0.2/unit1-1.0-SNAPSHOT.jar create mode 100644 modules/compute/src/test/resources/units/unit2/1.0.0/ignite-ut-job1-1.0-SNAPSHOT.jar delete mode 100644 modules/compute/src/test/resources/units/unit2/1.0.0/unit1-1.0-SNAPSHOT.jar create mode 100644 modules/compute/src/test/resources/units/unit2/2.0.0/ignite-ut-job2-1.0-SNAPSHOT.jar delete mode 100644 modules/compute/src/test/resources/units/unit2/2.0.0/unit2-1.0-SNAPSHOT.jar copy modules/metastorage/src/main/java/org/apache/ignite/internal/metastorage/command/{RemoveCommand.java => GetCurrentRevisionCommand.java} (75%) create mode 100644 modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/impl/MetaStorageManagerRecoveryTest.java create mode 100644 modules/platforms/cpp/ignite/client/compute/deployment_unit.h copy modules/platforms/dotnet/Apache.Ignite/{Internal/Proto/ResponseFlags.cs => Compute/DeploymentUnit.cs} (69%) create mode 100644 modules/runner/src/integrationTest/resources/units/ignite-it-jobs-1.0-SNAPSHOT.jar rename modules/runner/src/integrationTest/resources/units/{apache-ignite-1.0-SNAPSHOT-src.zip => ignite-jobs-1.0-SNAPSHOT-src.zip} (100%) delete mode 100644 modules/runner/src/integrationTest/resources/units/ignite-jobs-1.0-SNAPSHOT.jar create mode 100644 modules/runner/src/integrationTest/resources/units/ignite-ut-job1-1.0-SNAPSHOT.jar create mode 100644 modules/runner/src/integrationTest/resources/units/ignite-ut-job2-1.0-SNAPSHOT.jar delete mode 100644 modules/runner/src/integrationTest/resources/units/unit1-1.0-SNAPSHOT.jar delete mode 100644 modules/runner/src/integrationTest/resources/units/unit2-1.0-SNAPSHOT.jar rename modules/runner/src/{integrationTest => testFixtures}/java/org/apache/ignite/internal/Cluster.java (100%) rename modules/runner/src/{integrationTest => testFixtures}/java/org/apache/ignite/internal/sql/engine/util/CursorUtils.java (100%) rename modules/runner/src/{integrationTest => testFixtures}/java/org/apache/ignite/internal/sql/engine/util/MetadataMatcher.java (100%) rename modules/runner/src/{integrationTest => testFixtures}/java/org/apache/ignite/internal/sql/engine/util/NativeTypeWrapper.java (100%) rename modules/runner/src/{integrationTest => testFixtures}/java/org/apache/ignite/internal/sql/engine/util/QueryChecker.java (100%) rename modules/runner/src/{integrationTest => testFixtures}/java/org/apache/ignite/internal/sql/engine/util/TestQueryProcessor.java (100%) rename modules/runner/src/{integrationTest => testFixtures}/java/org/apache/ignite/internal/sql/engine/util/VarBinary.java (100%)