This is an automated email from the ASF dual-hosted git repository. sk0x50 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push: new e3c2d46 IGNITE-15881 Use direct properties in the TableManager. Fixes #431 e3c2d46 is described below commit e3c2d4613e2c69206b2f0e4e7d781c0817f11c5b Author: Alexander Polovtcev <alex.polovt...@gmail.com> AuthorDate: Tue Nov 9 19:06:39 2021 +0300 IGNITE-15881 Use direct properties in the TableManager. Fixes #431 Signed-off-by: Slava Koptilin <slava.kopti...@gmail.com> --- .../org/apache/ignite/internal/app/IgniteImpl.java | 1 - .../internal/table/distributed/TableManager.java | 84 ++------------- .../ignite/internal/table/TableManagerTest.java | 115 +++++---------------- 3 files changed, 34 insertions(+), 166 deletions(-) diff --git a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java index 7e71299..6c101d9 100644 --- a/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java +++ b/modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java @@ -206,7 +206,6 @@ public class IgniteImpl implements Ignite { raftMgr, baselineMgr, clusterSvc.topologyService(), - metaStorageMgr, getPartitionsStorePath(workDir) ); diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java index b720abb..5c8da9a 100644 --- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java +++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/TableManager.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -55,13 +54,10 @@ import org.apache.ignite.internal.configuration.schema.ExtendedTableChange; import org.apache.ignite.internal.configuration.schema.ExtendedTableConfiguration; import org.apache.ignite.internal.configuration.schema.ExtendedTableView; import org.apache.ignite.internal.configuration.schema.SchemaView; -import org.apache.ignite.internal.configuration.tree.NamedListNode; import org.apache.ignite.internal.configuration.util.ConfigurationUtil; import org.apache.ignite.internal.manager.EventListener; import org.apache.ignite.internal.manager.IgniteComponent; import org.apache.ignite.internal.manager.Producer; -import org.apache.ignite.internal.metastorage.MetaStorageManager; -import org.apache.ignite.internal.metastorage.client.Entry; import org.apache.ignite.internal.raft.Loza; import org.apache.ignite.internal.schema.SchemaDescriptor; import org.apache.ignite.internal.schema.SchemaUtils; @@ -79,10 +75,7 @@ import org.apache.ignite.internal.table.distributed.storage.InternalTableImpl; import org.apache.ignite.internal.table.event.TableEvent; import org.apache.ignite.internal.table.event.TableEventParameters; import org.apache.ignite.internal.util.ByteUtils; -import org.apache.ignite.internal.util.Cursor; import org.apache.ignite.internal.util.IgniteSpinBusyLock; -import org.apache.ignite.lang.ByteArray; -import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteException; import org.apache.ignite.lang.IgniteInternalCheckedException; import org.apache.ignite.lang.IgniteInternalException; @@ -109,19 +102,10 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp /** The logger. */ private static final IgniteLogger LOG = IgniteLogger.forClass(TableManager.class); - /** - * - */ + /** */ private static final int INITIAL_SCHEMA_VERSION = 1; - /** Public prefix for metastorage. */ - // TODO: IGNITE-15412 Remove after implementation. Configuration manager will be used to retrieve distributed values - // TODO: instead of metastorage manager. - private static final String PUBLIC_PREFIX = "dst-cfg.table.tables."; - - /** - * - */ + /** */ private static final IgniteUuidGenerator TABLE_ID_GENERATOR = new IgniteUuidGenerator(UUID.randomUUID(), 0); /** Tables configuration. */ @@ -136,11 +120,6 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp /** Baseline manager. */ private final BaselineManager baselineMgr; - // TODO: IGNITE-15412 Remove after implementation. Configuration manager will be used to retrieve distributed values - // TODO: instead of metastorage manager. - /** Meta storage service. */ - private final MetaStorageManager metaStorageMgr; - /** Storage engine instance. Only one type is available right now, which is the {@link RocksDbStorageEngine}. */ private final StorageEngine engine; @@ -163,7 +142,7 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp private final IgniteSpinBusyLock busyLock = new IgniteSpinBusyLock(); /** Prevents double stopping the component. */ - AtomicBoolean stopGuard = new AtomicBoolean(); + private final AtomicBoolean stopGuard = new AtomicBoolean(); /** * Creates a new table manager. @@ -172,7 +151,6 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp * @param dataStorageCfg Data storage configuration. * @param raftMgr Raft manager. * @param baselineMgr Baseline manager. - * @param metaStorageMgr Meta storage manager. * @param partitionsStoreDir Partitions store directory. */ public TableManager( @@ -181,14 +159,12 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp Loza raftMgr, BaselineManager baselineMgr, TopologyService topologyService, - MetaStorageManager metaStorageMgr, Path partitionsStoreDir ) { this.tablesCfg = tablesCfg; this.dataStorageCfg = dataStorageCfg; this.raftMgr = raftMgr; this.baselineMgr = baselineMgr; - this.metaStorageMgr = metaStorageMgr; this.partitionsStoreDir = partitionsStoreDir; netAddrResolver = addr -> { @@ -966,7 +942,7 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp * @return Future representing pending completion of the operation. */ private CompletableFuture<List<Table>> tablesAsyncInternal() { - var tableNames = tableNamesConfigured(); + List<String> tableNames = tableNamesConfigured(); var tableFuts = new CompletableFuture[tableNames.size()]; var i = 0; @@ -994,56 +970,12 @@ public class TableManager extends Producer<TableEvent, TableEventParameters> imp } /** - * Collects a set of table names from the distributed configuration storage. - * - * @return A set of table names. - */ - // TODO: IGNITE-15412 Configuration manager will be used to retrieve distributed values - // TODO: instead of metastorage manager. That will automatically resolve several bugs of current implementation. - private Set<String> tableNamesConfigured() { - IgniteBiTuple<ByteArray, ByteArray> range = toRange(new ByteArray(PUBLIC_PREFIX)); - - Set<String> tableNames = new HashSet<>(); - - try (Cursor<Entry> cursor = metaStorageMgr.range(range.get1(), range.get2())) { - while (cursor.hasNext()) { - Entry entry = cursor.next(); - - List<String> keySplit = ConfigurationUtil.split(entry.key().toString()); - - if (keySplit.size() == 5 && NamedListNode.NAME.equals(keySplit.get(4))) { - @Nullable byte[] value = entry.value(); - if (value != null) { - tableNames.add(ByteUtils.fromBytes(value).toString()); - } - } - } - } catch (Exception e) { - LOG.error("Can't get table names.", e); - } - - return tableNames; - } - - /** - * Transforms a prefix bytes to range. This method should be replaced to direct call of range by prefix in Meta storage manager when it - * will be implemented. + * Collects a list of table names from the distributed configuration storage. * - * @param prefixKey Prefix bytes. - * @return Tuple with left and right borders for range. + * @return A list of table names. */ - // TODO: IGNITE-15412 Remove after implementation. Configuration manager will be used to retrieve distributed values - // TODO: instead of metastorage manager. - private IgniteBiTuple<ByteArray, ByteArray> toRange(ByteArray prefixKey) { - var bytes = Arrays.copyOf(prefixKey.bytes(), prefixKey.bytes().length); - - if (bytes[bytes.length - 1] != Byte.MAX_VALUE) { - bytes[bytes.length - 1]++; - } else { - bytes = Arrays.copyOf(bytes, bytes.length + 1); - } - - return new IgniteBiTuple<>(prefixKey, new ByteArray(bytes)); + private List<String> tableNamesConfigured() { + return ConfigurationUtil.directValue(tablesCfg.tables()).namedListKeys(); } /** {@inheritDoc} */ diff --git a/modules/table/src/test/java/org/apache/ignite/internal/table/TableManagerTest.java b/modules/table/src/test/java/org/apache/ignite/internal/table/TableManagerTest.java index ebb1a23..0d4a80d 100644 --- a/modules/table/src/test/java/org/apache/ignite/internal/table/TableManagerTest.java +++ b/modules/table/src/test/java/org/apache/ignite/internal/table/TableManagerTest.java @@ -24,11 +24,9 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.times; @@ -42,7 +40,6 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Phaser; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import org.apache.ignite.configuration.schemas.store.DataStorageConfiguration; import org.apache.ignite.configuration.schemas.table.TableChange; @@ -52,18 +49,12 @@ import org.apache.ignite.internal.baseline.BaselineManager; import org.apache.ignite.internal.configuration.schema.ExtendedTableConfigurationSchema; import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension; import org.apache.ignite.internal.configuration.testframework.InjectConfiguration; -import org.apache.ignite.internal.configuration.tree.NamedListNode; -import org.apache.ignite.internal.metastorage.MetaStorageManager; -import org.apache.ignite.internal.metastorage.client.Entry; import org.apache.ignite.internal.raft.Loza; import org.apache.ignite.internal.schema.SchemaDescriptor; import org.apache.ignite.internal.schema.SchemaUtils; import org.apache.ignite.internal.schema.configuration.SchemaConfigurationConverter; import org.apache.ignite.internal.table.distributed.TableManager; import org.apache.ignite.internal.testframework.IgniteAbstractTest; -import org.apache.ignite.internal.util.ByteUtils; -import org.apache.ignite.internal.util.Cursor; -import org.apache.ignite.lang.ByteArray; import org.apache.ignite.lang.IgniteException; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.lang.IgniteUuidGenerator; @@ -95,9 +86,6 @@ import org.mockito.quality.Strictness; @ExtendWith({MockitoExtension.class, ConfigurationExtension.class}) @MockitoSettings(strictness = Strictness.LENIENT) public class TableManagerTest extends IgniteAbstractTest { - /** Public prefix for metastorage. */ - private static final String PUBLIC_PREFIX = "dst-cfg.table.tables."; - /** The name of the table which is statically configured. */ private static final String STATIC_TABLE_NAME = "t1"; @@ -114,22 +102,18 @@ public class TableManagerTest extends IgniteAbstractTest { private static final String NODE_NAME = "node1"; /** Count of replicas. */ - public static final int REPLICAS = 1; - - /** MetaStorage manager. */ - @Mock(lenient = true) - private MetaStorageManager mm; + private static final int REPLICAS = 1; /** Schema manager. */ - @Mock(lenient = true) + @Mock private BaselineManager bm; /** Topology service. */ - @Mock(lenient = true) + @Mock private TopologyService ts; /** Raft manager. */ - @Mock(lenient = true) + @Mock private Loza rm; /** Tables configuration. */ @@ -177,7 +161,6 @@ public class TableManagerTest extends IgniteAbstractTest { rm, bm, ts, - mm, workDir ); @@ -323,7 +306,7 @@ public class TableManagerTest extends IgniteAbstractTest { * Instantiates a table and prepares Table manager. */ @Test - public void testGetTableDuringCreation() throws Exception { + public void testGetTableDuringCreation() { TableDefinition scmTbl = SchemaBuilders.tableBuilder("PUBLIC", DYNAMIC_TABLE_FOR_DROP_NAME).columns( SchemaBuilders.column("key", ColumnType.INT64).asNonNull().build(), SchemaBuilders.column("val", ColumnType.INT64).asNullable().build() @@ -427,8 +410,6 @@ public class TableManagerTest extends IgniteAbstractTest { new NetworkAddress("localhost", 47500) )); - AtomicBoolean tableCreatedFlag = new AtomicBoolean(); - try (MockedStatic<SchemaUtils> schemaServiceMock = mockStatic(SchemaUtils.class)) { schemaServiceMock.when(() -> SchemaUtils.prepareSchemaDescriptor(anyInt(), any())) .thenReturn(mock(SchemaDescriptor.class)); @@ -447,78 +428,35 @@ public class TableManagerTest extends IgniteAbstractTest { TableManager tableManager = createTableManager(tblManagerFut); - TableImpl tbl2 = null; - - try { - when(mm.range(eq(new ByteArray(PUBLIC_PREFIX)), any())).thenAnswer(invocation -> { - Cursor<Entry> cursor = mock(Cursor.class); - - when(cursor.hasNext()).thenReturn(false); - - return cursor; - }); - - final int tablesBeforeCreation = tableManager.tables().size(); - - tblsCfg.tables().listen(ctx -> { - boolean createTbl = ctx.newValue().get(tableDefinition.canonicalName()) != null - && ctx.oldValue().get(tableDefinition.canonicalName()) == null; - - boolean dropTbl = ctx.oldValue().get(tableDefinition.canonicalName()) != null - && ctx.newValue().get(tableDefinition.canonicalName()) == null; - - if (!createTbl && !dropTbl) { - return CompletableFuture.completedFuture(null); - } - - tableCreatedFlag.set(createTbl); - - try { - when(mm.range(eq(new ByteArray(PUBLIC_PREFIX)), any())).thenAnswer(invocation -> { - AtomicBoolean firstRecord = new AtomicBoolean(createTbl); - - Cursor<Entry> cursor = mock(Cursor.class); + final int tablesBeforeCreation = tableManager.tables().size(); - when(cursor.hasNext()).thenAnswer(hasNextInvocation -> - firstRecord.compareAndSet(true, false)); + tblsCfg.tables().listen(ctx -> { + boolean createTbl = ctx.newValue().get(tableDefinition.canonicalName()) != null + && ctx.oldValue().get(tableDefinition.canonicalName()) == null; - Entry mockEntry = mock(Entry.class); - - when(mockEntry.key()).thenReturn(new ByteArray(PUBLIC_PREFIX + "uuid." + NamedListNode.NAME)); - - when(mockEntry.value()).thenReturn(ByteUtils.toBytes(tableDefinition.canonicalName())); - - when(cursor.next()).thenReturn(mockEntry); - - return cursor; - }); - } catch (NodeStoppingException e) { - log.error("Node was stopped during table creation.", e); - - fail(); - } - - if (phaser != null) { - phaser.arriveAndAwaitAdvance(); - } + boolean dropTbl = ctx.oldValue().get(tableDefinition.canonicalName()) != null + && ctx.newValue().get(tableDefinition.canonicalName()) == null; + if (!createTbl && !dropTbl) { return CompletableFuture.completedFuture(null); - }); + } - tbl2 = (TableImpl) tableManager.createTable(tableDefinition.canonicalName(), - tblCh -> SchemaConfigurationConverter.convert(tableDefinition, tblCh) - .changeReplicas(REPLICAS) - .changePartitions(PARTITIONS) - ); + if (phaser != null) { + phaser.arriveAndAwaitAdvance(); + } - assertNotNull(tbl2); + return CompletableFuture.completedFuture(null); + }); + + TableImpl tbl2 = (TableImpl) tableManager.createTable(tableDefinition.canonicalName(), + tblCh -> SchemaConfigurationConverter.convert(tableDefinition, tblCh) + .changeReplicas(REPLICAS) + .changePartitions(PARTITIONS) + ); - assertEquals(tablesBeforeCreation + 1, tableManager.tables().size()); - } catch (NodeStoppingException e) { - log.error("Node was stopped during table creation.", e); + assertNotNull(tbl2); - fail(); - } + assertEquals(tablesBeforeCreation + 1, tableManager.tables().size()); return tbl2; } @@ -537,7 +475,6 @@ public class TableManagerTest extends IgniteAbstractTest { rm, bm, ts, - mm, workDir );