This is an automated email from the ASF dual-hosted git repository.
tkalkirill pushed a commit to branch catalog-feature
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit ff97888bee55f28adc7b3465575bf4228194ebf4
Merge: 406b856f41 7989e7e253
Author: Kirill Tkalenko <tkalkir...@yandex.ru>
AuthorDate: Tue Aug 15 10:50:18 2023 +0300
Merge branch 'ai-main' into catalog-feature
.idea/inspectionProfiles/Project_Default.xml | 8 +-
build.gradle | 1 +
.../org/apache/ignite/table/mapper/Mapper.java | 3 +-
.../internal/binarytuple/BinaryTupleCommon.java | 6 -
.../handler/requests/jdbc/JdbcQueryCursor.java | 11 +-
.../requests/jdbc/JdbcQueryCursorSelfTest.java | 101 +++++
.../ignite/client/fakes/FakeIgniteTables.java | 22 +-
.../ignite/client/fakes/FakeInternalTable.java | 133 ++++---
.../ignite/client/fakes/FakeSchemaRegistry.java | 15 +-
.../BaseDistributionZoneManagerTest.java | 6 +-
.../DistributionZoneFiltersTest.java | 1 -
.../DistributionZoneManagerAlterFilterTest.java | 58 ++-
...ibutionZoneManagerConfigurationChangesTest.java | 59 ++-
.../DistributionZoneManagerFilterTest.java | 8 +-
.../DistributionZoneManagerScaleUpTest.java | 427 ++++++++++++---------
.../DistributionZoneManagerTest.java | 34 +-
.../DistributionZoneManagerWatchListenerTest.java | 17 +-
.../DistributionZoneCausalityDataNodesTest.java | 274 +++++++------
.../DistributionZoneRebalanceEngineTest.java | 58 ++-
.../DistributionZonesTestUtil.java | 39 ++
.../apache/ignite/internal/index/IndexManager.java | 49 ++-
.../ignite/internal/marshaller/Marshaller.java | 22 +-
modules/placement-driver/build.gradle | 1 +
.../MultiActorPlacementDriverTest.java | 36 +-
.../PlacementDriverManagerTest.java | 25 +-
.../Apache.Ignite.Tests/Compute/ComputeTests.cs | 10 +-
.../dotnet/Apache.Ignite.Tests/FakeServer.cs | 3 +-
.../dotnet/Apache.Ignite.Tests/IgniteTestsBase.cs | 6 +-
.../dotnet/Apache.Ignite.Tests/ListLogger.cs | 3 +-
.../Apache.Ignite.Tests/PartitionAwarenessTests.cs | 2 +-
.../Proto/ColocationHashTests.cs | 7 +-
.../dotnet/Apache.Ignite.Tests/RetryPolicyTests.cs | 8 +-
.../Sql/SqlResultSetObjectMappingTests.cs | 2 +-
.../Apache.Ignite.Tests/Table/DataStreamerTests.cs | 2 +-
.../CollectionExtensions.cs => Table/KeyPoco.cs} | 23 +-
.../Table/KeyValueViewBinaryTests.cs | 2 +-
.../Table/KeyValueViewPocoPrimitiveTests.cs | 118 +++---
.../Table/KeyValueViewPocoTests.cs | 123 +++---
.../Table/KeyValueViewPrimitivePocoTests.cs | 97 +++--
.../dotnet/Apache.Ignite.Tests/Table/PocoStruct.cs | 4 +-
.../Table/RecordViewBinaryTests.cs | 17 +-
.../Table/RecordViewCustomMappingTest.cs | 13 +-
.../Table/RecordViewPocoTests.cs | 2 +-
.../Table/RecordViewPrimitiveTests.cs | 3 +-
.../Table/SchemaSynchronizationTest.cs | 51 +--
.../Table/SchemaValidationTest.cs | 357 +++++++++++++++++
.../Table/Serialization/ReflectionUtilsTests.cs | 4 +-
.../CollectionExtensions.cs => Table/ValPoco.cs} | 26 +-
.../Transactions/TransactionsTests.cs | 2 +-
.../Internal}/Common/CollectionExtensions.cs | 24 +-
.../Internal/Linq/IgniteQueryExecutor.cs | 3 +-
.../Internal/Linq/IgniteQueryExpressionVisitor.cs | 8 +-
.../Apache.Ignite/Internal/Linq/ResultSelector.cs | 3 +-
.../Table/Serialization/ObjectSerializerHandler.cs | 91 ++++-
.../Table/Serialization/ReflectionUtils.cs | 3 +-
.../Serialization/TuplePairSerializerHandler.cs | 69 ++++
.../Table/Serialization/TupleSerializerHandler.cs | 42 ++
.../raft/server/ItSimpleCounterServerTest.java | 6 +-
...niteDistributionZoneManagerNodeRestartTest.java | 48 ++-
.../ItRaftCommandLeftInLogUntilRestartTest.java | 15 +-
.../internal/table/ItReadOnlyTransactionTest.java | 4 +-
.../ignite/internal/table/ItRoReadsTest.java | 53 +--
.../ignite/internal/table/ItTableScanTest.java | 8 +-
.../apache/ignite/internal/schema/BinaryRow.java | 3 -
.../ignite/internal/schema/BinaryRowConverter.java | 82 ++--
.../ignite/internal/schema/BinaryRowImpl.java | 7 -
.../{BinaryRow.java => ColumnsExtractor.java} | 30 +-
.../ignite/internal/schema/SchemaRegistry.java | 10 +-
.../marshaller/asm/AsmMarshallerGenerator.java | 5 +-
.../marshaller/reflection/KvMarshallerImpl.java | 17 +-
.../schema/marshaller/reflection/Marshaller.java | 28 +-
.../reflection/RecordMarshallerImpl.java | 4 +-
.../schema/registry/SchemaRegistryImpl.java | 58 ++-
.../schema/registry/UpgradingRowAdapter.java | 40 +-
.../org/apache/ignite/internal/schema/row/Row.java | 47 +--
.../ignite/internal/schema/row/RowAssembler.java | 44 +--
.../apache/ignite/internal/util/ObjectFactory.java | 2 +
.../benchmarks/SerializerBenchmarkTest.java | 7 +-
.../org/apache/ignite/internal/schema/RowTest.java | 2 +-
.../schema/marshaller/KvMarshallerTest.java | 68 ++--
.../schema/marshaller/RecordMarshallerTest.java | 12 +-
.../RecordMarshallerValidationsTest.java | 9 +-
.../sql/engine/exec/UpdatableTableImpl.java | 6 +-
.../internal/storage/BaseMvStoragesTest.java | 14 +-
.../ItInternalTableReadOnlyOperationsTest.java | 13 +-
.../ignite/distributed/ItTablePersistenceTest.java | 2 +-
.../distributed/ItTxDistributedTestSingleNode.java | 6 +-
.../ignite/internal/table/ItColocationTest.java | 2 +-
.../schema/marshaller/TupleMarshallerImpl.java | 29 +-
.../internal/table/AbstractRowTupleAdapter.java | 4 +-
.../apache/ignite/internal/table/IndexWrapper.java | 12 +-
.../internal/table/KeyValueBinaryViewImpl.java | 6 +-
.../ignite/internal/table/KeyValueViewImpl.java | 4 +-
.../internal/table/RecordBinaryViewImpl.java | 18 +-
.../ignite/internal/table/RecordViewImpl.java | 4 +-
.../apache/ignite/internal/table/TableImpl.java | 8 +-
.../table/distributed/HashIndexLocker.java | 20 +-
.../internal/table/distributed/IndexLocker.java | 18 +-
.../table/distributed/SortedIndexLocker.java | 21 +-
.../internal/table/distributed/TableManager.java | 17 +-
.../distributed/TableSchemaAwareIndexStorage.java | 21 +-
.../distributed/index/IndexUpdateHandler.java | 4 +-
.../request/MultipleRowReplicaRequest.java | 3 +-
.../replicator/PartitionReplicaListener.java | 111 +++---
.../table/ColocationHashCalculationTest.java | 2 +-
.../internal/table/distributed/IndexBaseTest.java | 39 +-
.../table/distributed/TableManagerTest.java | 8 +-
.../raft/PartitionCommandListenerTest.java | 22 +-
.../PartitionReplicaListenerIndexLockingTest.java | 91 +++--
.../replication/PartitionReplicaListenerTest.java | 159 +++++---
.../table/impl/DummyInternalTableImpl.java | 6 +-
.../table/impl/DummySchemaManagerImpl.java | 15 +-
112 files changed, 2476 insertions(+), 1360 deletions(-)
diff --cc
modules/index/src/main/java/org/apache/ignite/internal/index/IndexManager.java
index 9e8d54a95c,1f780838bd..a76702db28
---
a/modules/index/src/main/java/org/apache/ignite/internal/index/IndexManager.java
+++
b/modules/index/src/main/java/org/apache/ignite/internal/index/IndexManager.java
@@@ -28,10 -32,9 +28,9 @@@ import java.util.Objects
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
- import java.util.function.Function;
+import java.util.function.LongFunction;
import org.apache.ignite.configuration.NamedListView;
-import
org.apache.ignite.configuration.notifications.ConfigurationNamedListListener;
-import
org.apache.ignite.configuration.notifications.ConfigurationNotificationEvent;
+import org.apache.ignite.internal.catalog.CatalogManager;
import org.apache.ignite.internal.catalog.descriptors.CatalogColumnCollation;
import
org.apache.ignite.internal.catalog.descriptors.CatalogHashIndexDescriptor;
import
org.apache.ignite.internal.catalog.descriptors.CatalogIndexColumnDescriptor;
@@@ -382,108 -678,4 +401,108 @@@ public class IndexManager extends Produ
private static ColumnCollation toEventCollation(CatalogColumnCollation
collation) {
return ColumnCollation.get(collation.asc(), collation.nullsFirst());
}
+
+ private void startIndexes() {
+ CompletableFuture<Long> recoveryFinishedFuture =
metaStorageManager.recoveryFinishedFuture();
+
+ assert recoveryFinishedFuture.isDone();
+
+ int catalogVersion = catalogManager.latestCatalogVersion();
+ long causalityToken = recoveryFinishedFuture.join();
+
+ NamedListView<TableView> tableListView = tablesCfg.tables().value();
+
+ List<CompletableFuture<?>> startIndexFutures = new ArrayList<>();
+
+ for (CatalogIndexDescriptor index :
catalogManager.indexes(catalogVersion)) {
+ CatalogTableDescriptor table =
catalogManager.table(index.tableId(), catalogVersion);
+
+ assert table != null : "tableId=" + index.tableId() + ",
indexId=" + index.id();
+
+ // TODO: IGNITE-19499 Only catalog should be used
+ int configTableId = getConfigTableId(table.name(), tableListView);
+
+ CompletableFuture<?> fireCreateIndexEventFuture =
fireCreateIndexEvent(index, causalityToken, configTableId);
+
+ CompletableFuture<Void> registerIndexFuture =
registerIndex(table, index, causalityToken, configTableId);
+
+ startIndexFutures.add(allOf(fireCreateIndexEventFuture,
registerIndexFuture));
+ }
+
+ startVv.update(causalityToken, (unused, throwable) ->
allOf(startIndexFutures.toArray(CompletableFuture[]::new)))
+ .whenComplete((unused, throwable) -> {
+ if (throwable != null) {
+ LOG.error("Error starting indexes", throwable);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Indexes started successfully");
+ }
+ }
+ });
+ }
+
+ private CompletableFuture<Void> registerIndex(
+ CatalogTableDescriptor table,
+ CatalogIndexDescriptor index,
+ long causalityToken,
+ int configTableId
+ ) {
+ // TODO: IGNITE-19712 Listen to assignment changes and start new
index storages.
+ CompletableFuture<PartitionSet> tablePartitionFuture =
tableManager.localPartitionSetAsync(causalityToken, configTableId);
+
+ CompletableFuture<SchemaRegistry> schemaRegistryFuture =
schemaManager.schemaRegistry(causalityToken, configTableId);
+
+ return tablePartitionFuture.thenAcceptBoth(schemaRegistryFuture,
(partitionSet, schemaRegistry) -> {
+ TableImpl tableImpl = tableManager.getTable(configTableId);
+
+ assert tableImpl != null : "tableId=" + configTableId + ",
indexId=" + index.id();
+
+ var storageIndexDescriptor = StorageIndexDescriptor.create(table,
index);
+
+ TableRowToIndexKeyConverter tableRowConverter = new
TableRowToIndexKeyConverter(
+ schemaRegistry,
+
storageIndexDescriptor.columns().stream().map(StorageColumnDescriptor::name).toArray(String[]::new)
+ );
+
+ if (storageIndexDescriptor instanceof
StorageSortedIndexDescriptor) {
+ tableImpl.registerSortedIndex(
+ (StorageSortedIndexDescriptor) storageIndexDescriptor,
- tableRowConverter::convert,
++ tableRowConverter,
+ partitionSet
+ );
+ } else {
+ boolean unique = index.unique();
+
+ tableImpl.registerHashIndex(
+ (StorageHashIndexDescriptor) storageIndexDescriptor,
+ unique,
- tableRowConverter::convert,
++ tableRowConverter,
+ partitionSet
+ );
+
+ if (unique) {
+ tableImpl.pkId(index.id());
+ }
+ }
+ });
+ }
+
+ private CompletableFuture<?> fireCreateIndexEvent(
+ CatalogIndexDescriptor index,
+ long causalityToken,
+ int configTableId
+ ) {
+ return fireEvent(
+ IndexEvent.CREATE,
+ new IndexEventParameters(causalityToken, configTableId,
index.id(), toEventIndexDescriptor(index))
+ );
+ }
+
+ private static int getConfigTableId(String tableName,
NamedListView<TableView> tableListView) {
+ TableView tableView = tableListView.get(tableName);
+
+ assert tableView != null : tableName;
+
+ return tableView.id();
+ }
}
diff --cc
modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItRoReadsTest.java
index 30db632c93,99ef4e29ab..6124bea1c9
---
a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItRoReadsTest.java
+++
b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItRoReadsTest.java
@@@ -542,25 -556,6 +548,24 @@@ public class ItRoReadsTest extends Base
private static void assertRowEquals(BinaryRow row1, BinaryRow row2) {
assertThat(row1.schemaVersion(), is(row2.schemaVersion()));
- assertThat(row1.hasValue(), is(row2.hasValue()));
assertThat(row1.tupleSlice(), is(row2.tupleSlice()));
}
+
+ private static List<List<Object>> sql(String sql, Object... args) {
+ var queryEngine = new TestQueryProcessor(node());
+
+ SessionId sessionId =
queryEngine.createSession(PropertiesHelper.emptyHolder());
+
+ try {
+ var context = QueryContext.create(SqlQueryType.ALL);
+
+ CompletableFuture<AsyncSqlCursor<List<Object>>> queryFuture =
queryEngine.querySingleAsync(sessionId, context, sql, args);
+
+ assertThat(queryFuture, willCompleteSuccessfully());
+
+ return getAllFromCursor(queryFuture.join());
+ } finally {
+ queryEngine.closeSession(sessionId);
+ }
+ }
}
