This is an automated email from the ASF dual-hosted git repository. jshao pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push: new 27ca87535 [#3968] improvement(core): Disable KV entity store and optimize CI (#3975) 27ca87535 is described below commit 27ca87535e30535df2dc8f4108570671cd9e80bd Author: Qi Yu <y...@datastrato.com> AuthorDate: Mon Jul 8 19:49:18 2024 +0800 [#3968] improvement(core): Disable KV entity store and optimize CI (#3975) ### What changes were proposed in this pull request? - Disabling support for the KV entity store and adjusting the tests accordingly. - Change CI about backend option `jdbcBackend` ### Why are the changes needed? We are going to deprecate kv entity store Fix: #3968 ### Why are the changes needed? N/A ### How was this patch tested? Existing test. --- .github/workflows/backend-integration-test.yml | 2 +- build.gradle.kts | 5 +- catalogs/catalog-hadoop/build.gradle.kts | 10 + .../hadoop/TestHadoopCatalogOperations.java | 81 ++++++-- catalogs/catalog-kafka/build.gradle.kts | 10 + .../catalog/kafka/TestKafkaCatalogOperations.java | 69 ++++++- .../lakehouse/paimon/TestGravitinoPaimonTable.java | 11 ++ .../lakehouse/paimon/TestPaimonCatalog.java | 13 +- .../catalog/lakehouse/paimon/TestPaimonSchema.java | 13 ++ .../gravitino/filesystem/hadoop/TestGvfsBase.java | 8 + .../java/com/datastrato/gravitino/Configs.java | 2 +- .../datastrato/gravitino/EntityStoreFactory.java | 9 +- .../gravitino/storage/TestEntityStorage.java | 35 +--- .../storage/kv/TestEntityKeyEncoding.java | 2 + .../gravitino/storage/kv/TestKvEntityStorage.java | 1 + .../storage/kv/TestKvGarbageCollector.java | 2 + .../storage/kv/TestKvNameMappingService.java | 2 + .../gravitino/storage/kv/TestRocksDBKvBackend.java | 2 + .../gravitino/storage/kv/TestStorageVersion.java | 2 + .../storage/kv/TestTransactionIdGenerator.java | 1 + .../storage/kv/TestTransactionalKvBackend.java | 1 + .../storage/relational/TestJDBCBackend.java | 9 +- .../storage/relational/session/TestSqlSession.java | 8 +- core/src/test/resources/h2/schema-h2.sql | 218 --------------------- docs/gravitino-server-config.md | 2 +- gradle/libs.versions.toml | 1 + .../integration/test/util/AbstractIT.java | 2 +- .../relational/service/FilesetMetaServiceIT.java | 4 +- 28 files changed, 242 insertions(+), 283 deletions(-) diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index da534f467..437acbd02 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -61,7 +61,7 @@ jobs: architecture: [linux/amd64] java-version: [ 8, 11, 17 ] test-mode: [ embedded, deploy ] - backend: [ jdbcBackend, kvBackend] + backend: [ mysql, h2] env: PLATFORM: ${{ matrix.architecture }} steps: diff --git a/build.gradle.kts b/build.gradle.kts index ec5898921..a29b0405d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -178,9 +178,8 @@ allprojects { // Change poll image pause time from 30s to 60s param.environment("TESTCONTAINERS_PULL_PAUSE_TIMEOUT", "60") - if (project.hasProperty("jdbcBackend")) { - param.environment("jdbcBackend", "true") - } + val jdbcDatabase = project.properties["jdbcBackend"] as? String ?: "h2" + param.environment("jdbcBackend", jdbcDatabase) val testMode = project.properties["testMode"] as? String ?: "embedded" param.systemProperty("gravitino.log.path", project.buildDir.path + "/${project.name}-integration-test.log") diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index ccdf7c996..0dfa23154 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -53,6 +53,7 @@ dependencies { testImplementation(libs.bundles.log4j) testImplementation(libs.mockito.core) + testImplementation(libs.mockito.inline) testImplementation(libs.mysql.driver) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) @@ -101,6 +102,15 @@ tasks { } tasks.test { + doFirst { + val testMode = project.properties["testMode"] as? String ?: "embedded" + if (testMode == "deploy") { + environment("GRAVITINO_HOME", project.rootDir.path + "/distribution/package") + } else if (testMode == "embedded") { + environment("GRAVITINO_HOME", project.rootDir.path) + } + } + val skipUTs = project.hasProperty("skipTests") if (skipUTs) { // Only run integration tests diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 6fa41cd95..7023ad96e 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -18,20 +18,27 @@ */ package com.datastrato.gravitino.catalog.hadoop; -import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_KV_STORE; -import static com.datastrato.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; -import static com.datastrato.gravitino.Configs.ENTITY_KV_STORE; +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PATH; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_SERDE; import static com.datastrato.gravitino.Configs.ENTITY_STORE; +import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; import static com.datastrato.gravitino.Configs.STORE_DELETE_AFTER_TIME; import static com.datastrato.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME; +import static com.datastrato.gravitino.Configs.VERSION_RETENTION_COUNT; import static com.datastrato.gravitino.catalog.hadoop.HadoopCatalog.CATALOG_PROPERTIES_META; import static com.datastrato.gravitino.catalog.hadoop.HadoopCatalog.FILESET_PROPERTIES_META; import static com.datastrato.gravitino.catalog.hadoop.HadoopCatalog.SCHEMA_PROPERTIES_META; import static com.datastrato.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; import com.datastrato.gravitino.Config; -import com.datastrato.gravitino.Configs; -import com.datastrato.gravitino.EntitySerDeFactory; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.EntityStoreFactory; import com.datastrato.gravitino.NameIdentifier; @@ -49,9 +56,12 @@ import com.datastrato.gravitino.file.Fileset; import com.datastrato.gravitino.file.FilesetChange; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.storage.RandomIdGenerator; +import com.datastrato.gravitino.storage.relational.service.CatalogMetaService; +import com.datastrato.gravitino.storage.relational.service.MetalakeMetaService; import com.datastrato.gravitino.utils.NameIdentifierUtil; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import java.io.File; import java.io.IOException; import java.nio.file.Paths; import java.util.Arrays; @@ -71,13 +81,15 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.MockedStatic; import org.mockito.Mockito; public class TestHadoopCatalogOperations { - private static final String ROCKS_DB_STORE_PATH = + private static final String STORE_PATH = "/tmp/gravitino_test_entityStore_" + UUID.randomUUID().toString().replace("-", ""); + private static final String H2_file = STORE_PATH + ".mv.db"; private static final String UNFORMALIZED_TEST_ROOT_PATH = "/tmp/gravitino_test_catalog_" + UUID.randomUUID().toString().replace("-", ""); @@ -118,28 +130,65 @@ public class TestHadoopCatalogOperations { @BeforeAll public static void setUp() { Config config = Mockito.mock(Config.class); - Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv"); - Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE); - Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto"); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(ROCKS_DB_STORE_PATH); - - Assertions.assertEquals(ROCKS_DB_STORE_PATH, config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)); - Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); - Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); + when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); + when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); + when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_PATH)).thenReturn(STORE_PATH); + + // The following properties are used to create the JDBC connection; they are just for test, in + // the real world, + // they will be set automatically by the configuration file if you set ENTITY_RELATIONAL_STORE + // as EMBEDDED_ENTITY_RELATIONAL_STORE. + when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL)) + .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", STORE_PATH)); + when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("gravitino"); + when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)).thenReturn("gravitino"); + when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)).thenReturn("org.h2.Driver"); + + File f = FileUtils.getFile(STORE_PATH); + f.deleteOnExit(); + + when(config.get(VERSION_RETENTION_COUNT)).thenReturn(1L); + when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); + when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); + when(config.get(ENTITY_SERDE)).thenReturn("proto"); store = EntityStoreFactory.createEntityStore(config); store.initialize(config); - store.setSerDe(EntitySerDeFactory.createEntitySerDe(config)); idGenerator = new RandomIdGenerator(); + + // Mock + MetalakeMetaService metalakeMetaService = MetalakeMetaService.getInstance(); + MetalakeMetaService spyMetaservice = Mockito.spy(metalakeMetaService); + doReturn(1L).when(spyMetaservice).getMetalakeIdByName(Mockito.anyString()); + + CatalogMetaService catalogMetaService = CatalogMetaService.getInstance(); + CatalogMetaService spyCatalogMetaService = Mockito.spy(catalogMetaService); + doReturn(1L) + .when(spyCatalogMetaService) + .getCatalogIdByMetalakeIdAndName(Mockito.anyLong(), Mockito.anyString()); + + MockedStatic<MetalakeMetaService> metalakeMetaServiceMockedStatic = + Mockito.mockStatic(MetalakeMetaService.class); + MockedStatic<CatalogMetaService> catalogMetaServiceMockedStatic = + Mockito.mockStatic(CatalogMetaService.class); + + metalakeMetaServiceMockedStatic + .when(MetalakeMetaService::getInstance) + .thenReturn(spyMetaservice); + catalogMetaServiceMockedStatic + .when(CatalogMetaService::getInstance) + .thenReturn(spyCatalogMetaService); } @AfterAll public static void tearDown() throws IOException { store.close(); - FileUtils.deleteDirectory(FileUtils.getFile(ROCKS_DB_STORE_PATH)); new Path(TEST_ROOT_PATH) .getFileSystem(new Configuration()) .delete(new Path(TEST_ROOT_PATH), true); + + File f = FileUtils.getFile(H2_file); + f.delete(); } @Test diff --git a/catalogs/catalog-kafka/build.gradle.kts b/catalogs/catalog-kafka/build.gradle.kts index 790383433..8ce72eeba 100644 --- a/catalogs/catalog-kafka/build.gradle.kts +++ b/catalogs/catalog-kafka/build.gradle.kts @@ -43,6 +43,7 @@ dependencies { testImplementation(libs.junit.jupiter.api) testImplementation(libs.kafka) testImplementation(libs.mockito.core) + testImplementation(libs.mockito.inline) testImplementation(libs.mysql.driver) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) @@ -83,6 +84,15 @@ tasks.getByName("generateMetadataFileForMavenJavaPublication") { } tasks.test { + doFirst { + val testMode = project.properties["testMode"] as? String ?: "embedded" + if (testMode == "deploy") { + environment("GRAVITINO_HOME", project.rootDir.path + "/distribution/package") + } else if (testMode == "embedded") { + environment("GRAVITINO_HOME", project.rootDir.path) + } + } + val skipUTs = project.hasProperty("skipTests") if (skipUTs) { // Only run integration tests diff --git a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java index 2d29b2f68..93cee22d2 100644 --- a/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java +++ b/catalogs/catalog-kafka/src/test/java/com/datastrato/gravitino/catalog/kafka/TestKafkaCatalogOperations.java @@ -20,11 +20,21 @@ package com.datastrato.gravitino.catalog.kafka; import static com.datastrato.gravitino.Catalog.Type.MESSAGING; import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_KV_STORE; +import static com.datastrato.gravitino.Configs.DEFAULT_ENTITY_RELATIONAL_STORE; import static com.datastrato.gravitino.Configs.ENTITY_KV_ROCKSDB_BACKEND_PATH; import static com.datastrato.gravitino.Configs.ENTITY_KV_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_PATH; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_URL; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_JDBC_BACKEND_USER; +import static com.datastrato.gravitino.Configs.ENTITY_RELATIONAL_STORE; +import static com.datastrato.gravitino.Configs.ENTITY_SERDE; import static com.datastrato.gravitino.Configs.ENTITY_STORE; +import static com.datastrato.gravitino.Configs.RELATIONAL_ENTITY_STORE; import static com.datastrato.gravitino.Configs.STORE_DELETE_AFTER_TIME; import static com.datastrato.gravitino.Configs.STORE_TRANSACTION_MAX_SKEW_TIME; +import static com.datastrato.gravitino.Configs.VERSION_RETENTION_COUNT; import static com.datastrato.gravitino.StringIdentifier.ID_KEY; import static com.datastrato.gravitino.catalog.kafka.KafkaCatalog.CATALOG_PROPERTIES_METADATA; import static com.datastrato.gravitino.catalog.kafka.KafkaCatalog.SCHEMA_PROPERTIES_METADATA; @@ -33,10 +43,11 @@ import static com.datastrato.gravitino.catalog.kafka.KafkaCatalogOperations.CLIE import static com.datastrato.gravitino.catalog.kafka.KafkaCatalogPropertiesMetadata.BOOTSTRAP_SERVERS; import static com.datastrato.gravitino.catalog.kafka.KafkaTopicPropertiesMetadata.PARTITION_COUNT; import static com.datastrato.gravitino.catalog.kafka.KafkaTopicPropertiesMetadata.REPLICATION_FACTOR; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.when; import com.datastrato.gravitino.Config; import com.datastrato.gravitino.Configs; -import com.datastrato.gravitino.EntitySerDeFactory; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.EntityStoreFactory; import com.datastrato.gravitino.NameIdentifier; @@ -55,7 +66,10 @@ import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.CatalogEntity; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.storage.RandomIdGenerator; +import com.datastrato.gravitino.storage.relational.service.CatalogMetaService; +import com.datastrato.gravitino.storage.relational.service.MetalakeMetaService; import com.google.common.collect.ImmutableMap; +import java.io.File; import java.io.IOException; import java.time.Instant; import java.util.Map; @@ -65,12 +79,13 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; import org.mockito.Mockito; public class TestKafkaCatalogOperations extends KafkaClusterEmbedded { - private static final String ROCKS_DB_STORE_PATH = - "/tmp/gravitino_test_entityStore_" + genRandomString(); + private static final String STORE_PATH = "/tmp/gravitino_test_entityStore_" + genRandomString(); + private static final String H2_FILE = STORE_PATH + ".mv.db"; private static final String METALAKE_NAME = "metalake"; private static final String CATALOG_NAME = "test_kafka_catalog"; private static final String DEFAULT_SCHEMA_NAME = "default"; @@ -114,15 +129,55 @@ public class TestKafkaCatalogOperations extends KafkaClusterEmbedded { Mockito.when(config.get(ENTITY_STORE)).thenReturn("kv"); Mockito.when(config.get(ENTITY_KV_STORE)).thenReturn(DEFAULT_ENTITY_KV_STORE); Mockito.when(config.get(Configs.ENTITY_SERDE)).thenReturn("proto"); - Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(ROCKS_DB_STORE_PATH); + Mockito.when(config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)).thenReturn(STORE_PATH); - Assertions.assertEquals(ROCKS_DB_STORE_PATH, config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)); + Assertions.assertEquals(STORE_PATH, config.get(ENTITY_KV_ROCKSDB_BACKEND_PATH)); Mockito.when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); Mockito.when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); + when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE); + when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE); + when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_PATH)).thenReturn(STORE_PATH); + + when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_URL)) + .thenReturn(String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", STORE_PATH)); + when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_USER)).thenReturn("gravitino"); + when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_PASSWORD)).thenReturn("gravitino"); + when(config.get(ENTITY_RELATIONAL_JDBC_BACKEND_DRIVER)).thenReturn("org.h2.Driver"); + + File f = FileUtils.getFile(STORE_PATH); + f.deleteOnExit(); + + when(config.get(VERSION_RETENTION_COUNT)).thenReturn(1L); + when(config.get(STORE_TRANSACTION_MAX_SKEW_TIME)).thenReturn(1000L); + when(config.get(STORE_DELETE_AFTER_TIME)).thenReturn(20 * 60 * 1000L); + when(config.get(ENTITY_SERDE)).thenReturn("proto"); + + // Mock + MetalakeMetaService metalakeMetaService = MetalakeMetaService.getInstance(); + MetalakeMetaService spyMetaservice = Mockito.spy(metalakeMetaService); + doReturn(1L).when(spyMetaservice).getMetalakeIdByName(Mockito.anyString()); + + CatalogMetaService catalogMetaService = CatalogMetaService.getInstance(); + CatalogMetaService spyCatalogMetaService = Mockito.spy(catalogMetaService); + doReturn(1L) + .when(spyCatalogMetaService) + .getCatalogIdByMetalakeIdAndName(Mockito.anyLong(), Mockito.anyString()); + + MockedStatic<MetalakeMetaService> metalakeMetaServiceMockedStatic = + Mockito.mockStatic(MetalakeMetaService.class); + MockedStatic<CatalogMetaService> catalogMetaServiceMockedStatic = + Mockito.mockStatic(CatalogMetaService.class); + + metalakeMetaServiceMockedStatic + .when(MetalakeMetaService::getInstance) + .thenReturn(spyMetaservice); + catalogMetaServiceMockedStatic + .when(CatalogMetaService::getInstance) + .thenReturn(spyCatalogMetaService); + store = EntityStoreFactory.createEntityStore(config); store.initialize(config); - store.setSerDe(EntitySerDeFactory.createEntitySerDe(config)); idGenerator = new RandomIdGenerator(); kafkaCatalogEntity = CatalogEntity.builder() @@ -147,7 +202,7 @@ public class TestKafkaCatalogOperations extends KafkaClusterEmbedded { public static void tearDown() throws IOException { if (store != null) { store.close(); - FileUtils.deleteDirectory(FileUtils.getFile(ROCKS_DB_STORE_PATH)); + FileUtils.deleteQuietly(FileUtils.getFile(H2_FILE)); } } diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java index eba62d4de..1c8b40fe6 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestGravitinoPaimonTable.java @@ -38,11 +38,15 @@ import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.Maps; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; import java.time.Instant; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.UUID; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.paimon.schema.Schema; import org.apache.paimon.types.DataField; @@ -98,6 +102,13 @@ public class TestGravitinoPaimonTable { @AfterAll static void cleanUp() { paimonCatalogOperations.dropSchema(schemaIdent, true); + String warehousePath = "/tmp/paimon_catalog_warehouse"; + try { + FileUtils.deleteDirectory(new File(warehousePath)); + Files.delete(Paths.get(warehousePath)); + } catch (Exception e) { + // Ignore + } } private static CatalogEntity createDefaultCatalogEntity() { diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java index fb8cb2bd6..3ba2dde0c 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java @@ -35,6 +35,8 @@ import java.io.File; import java.io.IOException; import java.time.Instant; import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -69,9 +71,18 @@ public class TestPaimonCatalog { } }; - private String tempDir = + private static String tempDir = String.join(File.separator, System.getProperty("java.io.tmpdir"), "paimon_catalog_warehouse"); + @AfterAll + public static void clean() { + try { + FileUtils.deleteDirectory(new File(tempDir)); + } catch (Exception e) { + // Ignore + } + } + @Test public void testCatalogOperation() { AuditInfo auditInfo = diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonSchema.java b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonSchema.java index f492c30ab..3fe4f563c 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonSchema.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/com/datastrato/gravitino/catalog/lakehouse/paimon/TestPaimonSchema.java @@ -28,11 +28,14 @@ import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.CatalogEntity; import com.google.common.collect.Maps; import java.io.File; +import java.io.IOException; import java.time.Instant; import java.util.Arrays; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -51,6 +54,16 @@ public class TestPaimonSchema { NoSuchSchemaException.class, () -> paimonCatalogOperations.loadSchema(ident)); } + @AfterAll + static void cleanUpFile() { + try { + FileUtils.deleteDirectory( + new File(System.getProperty("java.io.tmpdir") + "/paimon_catalog_warehouse")); + } catch (IOException e) { + // Ignore + } + } + @Test public void testCreatePaimonSchema() { PaimonCatalog paimonCatalog = initPaimonCatalog("testCreatePaimonSchema"); diff --git a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java index dfe489421..2c6af2ef5 100644 --- a/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java +++ b/clients/filesystem-hadoop3/src/test/java/com/datastrato/gravitino/filesystem/hadoop/TestGvfsBase.java @@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.file.Fileset; +import java.io.File; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -37,6 +38,7 @@ import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -80,6 +82,12 @@ public class TestGvfsBase extends GravitinoMockServerBase { @AfterAll public static void tearDown() { GravitinoMockServerBase.tearDown(); + String fileName = FileSystemTestUtils.localRootPrefix().replace("file:", ""); + try { + FileUtils.deleteDirectory(new File(fileName)); + } catch (Exception e) { + // Ignore + } } @BeforeEach diff --git a/core/src/main/java/com/datastrato/gravitino/Configs.java b/core/src/main/java/com/datastrato/gravitino/Configs.java index 5ee2232ca..6c7fa3fb5 100644 --- a/core/src/main/java/com/datastrato/gravitino/Configs.java +++ b/core/src/main/java/com/datastrato/gravitino/Configs.java @@ -30,7 +30,7 @@ public class Configs { private Configs() {} - public static final String DEFAULT_ENTITY_STORE = "kv"; + public static final String KV_STORE_KEY = "kv"; public static final String RELATIONAL_ENTITY_STORE = "relational"; public static final String ENTITY_STORE_KEY = "gravitino.entity.store"; diff --git a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java index 249a0a9e9..bfd74efc7 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java +++ b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java @@ -18,6 +18,8 @@ */ package com.datastrato.gravitino; +import static com.datastrato.gravitino.Configs.KV_STORE_KEY; + import com.datastrato.gravitino.storage.kv.KvEntityStore; import com.datastrato.gravitino.storage.relational.RelationalEntityStore; import com.google.common.collect.ImmutableMap; @@ -36,7 +38,7 @@ public class EntityStoreFactory { // doesn't need to specify the full qualified class name when creating an EntityStore. public static final ImmutableMap<String, String> ENTITY_STORES = ImmutableMap.of( - Configs.DEFAULT_ENTITY_STORE, + KV_STORE_KEY, KvEntityStore.class.getCanonicalName(), Configs.RELATIONAL_ENTITY_STORE, RelationalEntityStore.class.getCanonicalName()); @@ -54,6 +56,11 @@ public class EntityStoreFactory { String name = config.get(Configs.ENTITY_STORE); String className = ENTITY_STORES.getOrDefault(name, name); + if (KV_STORE_KEY.equals(name)) { + throw new UnsupportedOperationException( + "KvEntityStore is not supported since version 0.6.0. Please use RelationalEntityStore instead."); + } + try { return (EntityStore) Class.forName(className).getDeclaredConstructor().newInstance(); } catch (Exception e) { diff --git a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java index b38dc0ef5..09a14682c 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/TestEntityStorage.java @@ -63,14 +63,12 @@ import com.datastrato.gravitino.meta.SchemaVersion; import com.datastrato.gravitino.meta.TableEntity; import com.datastrato.gravitino.meta.TopicEntity; import com.datastrato.gravitino.meta.UserEntity; -import com.datastrato.gravitino.storage.relational.RelationalEntityStore; import com.datastrato.gravitino.storage.relational.session.SqlSessionFactoryHelper; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -78,10 +76,8 @@ import java.sql.Statement; import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.UUID; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.ibatis.session.SqlSession; import org.junit.jupiter.api.Assertions; @@ -96,14 +92,15 @@ public class TestEntityStorage { private static final String JDBC_STORE_PATH = "/tmp/gravitino_jdbc_entityStore_" + UUID.randomUUID().toString().replace("-", ""); private static final String DB_DIR = JDBC_STORE_PATH + "/testdb"; + private static final String H2_FILE = DB_DIR + ".mv.db"; static Object[] storageProvider() { - return new Object[] {Configs.DEFAULT_ENTITY_STORE, Configs.RELATIONAL_ENTITY_STORE}; + return new Object[] {Configs.RELATIONAL_ENTITY_STORE}; } private void init(String type, Config config) { Preconditions.checkArgument(StringUtils.isNotBlank(type)); - if (type.equals(Configs.DEFAULT_ENTITY_STORE)) { + if (type.equals(Configs.KV_STORE_KEY)) { try { FileUtils.deleteDirectory(FileUtils.getFile(KV_STORE_PATH)); } catch (Exception e) { @@ -145,28 +142,9 @@ public class TestEntityStorage { } } - private void prepareJdbcTable() { - // Read the ddl sql to create table - String scriptPath = "h2/schema-h2.sql"; - try (SqlSession sqlSession = - SqlSessionFactoryHelper.getInstance().getSqlSessionFactory().openSession(true); - Connection connection = sqlSession.getConnection(); - Statement statement = connection.createStatement()) { - StringBuilder ddlBuilder = new StringBuilder(); - IOUtils.readLines( - Objects.requireNonNull( - this.getClass().getClassLoader().getResourceAsStream(scriptPath)), - StandardCharsets.UTF_8) - .forEach(line -> ddlBuilder.append(line).append("\n")); - statement.execute(ddlBuilder.toString()); - } catch (Exception e) { - throw new IllegalStateException("Create tables failed", e); - } - } - private void destroy(String type) { Preconditions.checkArgument(StringUtils.isNotBlank(type)); - if (type.equals(Configs.DEFAULT_ENTITY_STORE)) { + if (type.equals(Configs.KV_STORE_KEY)) { try { FileUtils.deleteDirectory(FileUtils.getFile(KV_STORE_PATH)); } catch (Exception e) { @@ -178,6 +156,8 @@ public class TestEntityStorage { if (dir.exists()) { dir.delete(); } + + FileUtils.deleteQuietly(new File(H2_FILE)); } else { throw new UnsupportedOperationException("Unsupported entity store type: " + type); } @@ -2135,9 +2115,6 @@ public class TestEntityStorage { try (EntityStore store = EntityStoreFactory.createEntityStore(config)) { store.initialize(config); - if (store instanceof RelationalEntityStore) { - prepareJdbcTable(); - } BaseMetalake metalake = createBaseMakeLake(1L, "metalake", auditInfo); CatalogEntity catalog = createCatalog(1L, Namespace.of("metalake"), "catalog", auditInfo); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java index 588bde8e6..a8c7a2ac3 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java @@ -46,6 +46,7 @@ import java.util.Random; import java.util.stream.Stream; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; @@ -55,6 +56,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; @TestInstance(Lifecycle.PER_CLASS) +@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") public class TestEntityKeyEncoding { private Config getConfig() throws IOException { File baseDir = new File(System.getProperty("java.io.tmpdir")); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvEntityStorage.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvEntityStorage.java index f8d41f351..28985c31e 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvEntityStorage.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvEntityStorage.java @@ -63,6 +63,7 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") public class TestKvEntityStorage extends TestEntityStorage { @BeforeEach @AfterEach diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java index 8a58f8525..cef6e2e40 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvGarbageCollector.java @@ -59,10 +59,12 @@ import java.time.Instant; import java.util.List; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @SuppressWarnings("DefaultCharset") +@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") class TestKvGarbageCollector { public Config getConfig() throws IOException { Config config = Mockito.mock(Config.class); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java index 40a6bacc8..f8cdd091b 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java @@ -37,9 +37,11 @@ import java.io.IOException; import java.lang.reflect.Field; import java.nio.file.Files; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") public class TestKvNameMappingService { private Config getConfig() throws IOException { File baseDir = new File(System.getProperty("java.io.tmpdir")); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestRocksDBKvBackend.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestRocksDBKvBackend.java index 588908ba2..e1d3d80b0 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestRocksDBKvBackend.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestRocksDBKvBackend.java @@ -30,11 +30,13 @@ import java.nio.file.Files; import java.util.List; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; +@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") public class TestRocksDBKvBackend { private KvBackend getKvBackEnd() throws IOException { diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestStorageVersion.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestStorageVersion.java index 3f8bd230b..fcf1eb03d 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestStorageVersion.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestStorageVersion.java @@ -37,9 +37,11 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") class TestStorageVersion { @Test diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionIdGenerator.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionIdGenerator.java index a2903bc4a..22e6b9f5e 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionIdGenerator.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionIdGenerator.java @@ -42,6 +42,7 @@ import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") public class TestTransactionIdGenerator { private static final Logger LOGGER = LoggerFactory.getLogger(TestTransactionalKvBackend.class); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionalKvBackend.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionalKvBackend.java index 936fbc492..7514c290c 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionalKvBackend.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestTransactionalKvBackend.java @@ -54,6 +54,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; @SuppressWarnings("DefaultCharset") +@Disabled("Gravitino will not support KV entity store since 0.6.0, so we disable this test.") class TestTransactionalKvBackend { private static final Logger LOGGER = LoggerFactory.getLogger(TestTransactionalKvBackend.class); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java index ffb7f9dc4..7a62eacab 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/TestJDBCBackend.java @@ -65,6 +65,8 @@ import com.google.common.collect.Lists; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -88,6 +90,7 @@ public class TestJDBCBackend { private static final String JDBC_STORE_PATH = "/tmp/gravitino_jdbc_entityStore_" + UUID.randomUUID().toString().replace("-", ""); private static final String DB_DIR = JDBC_STORE_PATH + "/testdb"; + private static final String H2_FILE = DB_DIR + ".mv.db"; private static final Config config = Mockito.mock(Config.class); public static final ImmutableMap<String, String> RELATIONAL_BACKENDS = ImmutableMap.of( @@ -97,6 +100,7 @@ public class TestJDBCBackend { @BeforeAll public static void setup() { File dir = new File(DB_DIR); + dir.deleteOnExit(); if (dir.exists() || !dir.isDirectory()) { dir.delete(); } @@ -127,8 +131,11 @@ public class TestJDBCBackend { dropAllTables(); File dir = new File(DB_DIR); if (dir.exists()) { - dir.delete(); + Files.delete(Paths.get(DB_DIR)); } + + Files.delete(Paths.get(H2_FILE)); + Files.delete(Paths.get(JDBC_STORE_PATH)); backend.close(); } diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java index e64e454c1..03e9904ed 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/session/TestSqlSession.java @@ -35,9 +35,12 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.datastrato.gravitino.Config; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.sql.SQLException; import java.util.UUID; import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.io.FileUtils; import org.apache.ibatis.session.SqlSession; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -85,8 +88,11 @@ public class TestSqlSession { public static void tearDown() throws IOException { File dir = new File(DB_DIR); if (dir.exists()) { - dir.delete(); + FileUtils.deleteDirectory(FileUtils.getFile(DB_DIR)); } + + Files.delete(Paths.get(MYSQL_STORE_PATH)); + SqlSessionFactoryHelper.getInstance().close(); } diff --git a/core/src/test/resources/h2/schema-h2.sql b/core/src/test/resources/h2/schema-h2.sql deleted file mode 100644 index 8d29af599..000000000 --- a/core/src/test/resources/h2/schema-h2.sql +++ /dev/null @@ -1,218 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file-- --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"). You may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, --- software distributed under the License is distributed on an --- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --- KIND, either express or implied. See the License for the --- specific language governing permissions and limitations --- under the License. --- - -CREATE TABLE IF NOT EXISTS `metalake_meta` ( - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', - `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info', - `schema_version` MEDIUMTEXT NOT NULL COMMENT 'metalake schema version info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'metalake deleted at', - PRIMARY KEY (metalake_id), - CONSTRAINT uk_mn_del UNIQUE (metalake_name, deleted_at) -) ENGINE = InnoDB; - - -CREATE TABLE IF NOT EXISTS `catalog_meta` ( - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `catalog_name` VARCHAR(128) NOT NULL COMMENT 'catalog name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `type` VARCHAR(64) NOT NULL COMMENT 'catalog type', - `provider` VARCHAR(64) NOT NULL COMMENT 'catalog provider', - `catalog_comment` VARCHAR(256) DEFAULT '' COMMENT 'catalog comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'catalog properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'catalog audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'catalog deleted at', - PRIMARY KEY (catalog_id), - CONSTRAINT uk_mid_cn_del UNIQUE (metalake_id, catalog_name, deleted_at) -) ENGINE=InnoDB; - - -CREATE TABLE IF NOT EXISTS `schema_meta` ( - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `schema_name` VARCHAR(128) NOT NULL COMMENT 'schema name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_comment` VARCHAR(256) DEFAULT '' COMMENT 'schema comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'schema audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'schema current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'schema last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'schema deleted at', - PRIMARY KEY (schema_id), - CONSTRAINT uk_cid_sn_del UNIQUE (catalog_id, schema_name, deleted_at), - -- Aliases are used here, and indexes with the same name in H2 can only be created once. - KEY idx_smid (metalake_id) -) ENGINE=InnoDB; - -CREATE TABLE IF NOT EXISTS `table_meta` ( - `table_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'table id', - `table_name` VARCHAR(128) NOT NULL COMMENT 'table name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'table audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'table current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'table last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'table deleted at', - PRIMARY KEY (table_id), - CONSTRAINT uk_sid_tn_del UNIQUE (schema_id, table_name, deleted_at), - -- Aliases are used here, and indexes with the same name in H2 can only be created once. - KEY idx_tmid (metalake_id), - KEY idx_tcid (catalog_id) -) ENGINE=InnoDB; - - -CREATE TABLE IF NOT EXISTS `fileset_meta` ( - `fileset_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'fileset id', - `fileset_name` VARCHAR(128) NOT NULL COMMENT 'fileset name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `type` VARCHAR(64) NOT NULL COMMENT 'fileset type', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'fileset audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'fileset current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'fileset last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'fileset deleted at', - PRIMARY KEY (fileset_id), - CONSTRAINT uk_sid_fn_del UNIQUE (schema_id, fileset_name, deleted_at), - -- Aliases are used here, and indexes with the same name in H2 can only be created once. - KEY idx_fmid (metalake_id), - KEY idx_fcid (catalog_id) -) ENGINE=InnoDB; - - -CREATE TABLE IF NOT EXISTS `fileset_version_info` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `fileset_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'fileset id', - `version` INT UNSIGNED NOT NULL COMMENT 'fileset info version', - `fileset_comment` VARCHAR(256) DEFAULT '' COMMENT 'fileset comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'fileset properties', - `storage_location` MEDIUMTEXT DEFAULT NULL COMMENT 'fileset storage location', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'fileset deleted at', - PRIMARY KEY (id), - CONSTRAINT uk_fid_ver_del UNIQUE (fileset_id, version, deleted_at), - -- Aliases are used here, and indexes with the same name in H2 can only be created once. - KEY idx_fvmid (metalake_id), - KEY idx_fvcid (catalog_id) -) ENGINE=InnoDB; - -CREATE TABLE IF NOT EXISTS `topic_meta` ( - `topic_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'topic id', - `topic_name` VARCHAR(128) NOT NULL COMMENT 'topic name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `comment` VARCHAR(256) DEFAULT '' COMMENT 'topic comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'topic properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'topic audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'topic current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'topic last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'topic deleted at', - PRIMARY KEY (topic_id), - CONSTRAINT uk_cid_tn_del UNIQUE (schema_id, topic_name, deleted_at), - -- Aliases are used here, and indexes with the same name in H2 can only be created once. - KEY idx_tvmid (metalake_id), - KEY idx_tvcid (catalog_id) -) ENGINE=InnoDB; - -CREATE TABLE IF NOT EXISTS `user_meta` ( - `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', - `user_name` VARCHAR(128) NOT NULL COMMENT 'username', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'user audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'user deleted at', - PRIMARY KEY (`user_id`), - CONSTRAINT `uk_mid_us_del` UNIQUE (`metalake_id`, `user_name`, `deleted_at`) -) ENGINE=InnoDB; - -CREATE TABLE IF NOT EXISTS `role_meta` ( - `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', - `role_name` VARCHAR(128) NOT NULL COMMENT 'role name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'role audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'role deleted at', - PRIMARY KEY (`role_id`), - CONSTRAINT `uk_mid_rn_del` UNIQUE (`metalake_id`, `role_name`, `deleted_at`) -) ENGINE=InnoDB; - -CREATE TABLE IF NOT EXISTS `role_meta_securable_object` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', - `entity_id` BIGINT(20) NOT NULL COMMENT 'securable object entity id', - `type` VARCHAR(128) NOT NULL COMMENT 'securable object type', - `privilege_names` VARCHAR(256) NOT NULL COMMENT 'securable object privilege names', - `privilege_conditions` VARCHAR(256) NOT NULL COMMENT 'securable object privilege conditions', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable objectcurrent version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable object last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'securable object deleted at', - PRIMARY KEY (`id`), - KEY `idx_obj_rid` (`role_id`) - ) ENGINE=InnoDB; - -CREATE TABLE IF NOT EXISTS `user_role_rel` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', - `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation deleted at', - PRIMARY KEY (`id`), - CONSTRAINT `uk_ui_ri_del` UNIQUE (`user_id`, `role_id`, `deleted_at`), - KEY `idx_rid` (`role_id`) -) ENGINE=InnoDB; - -CREATE TABLE IF NOT EXISTS `group_meta` ( - `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id', - `group_name` VARCHAR(128) NOT NULL COMMENT 'group name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'group audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'group deleted at', - PRIMARY KEY (`group_id`), - CONSTRAINT `uk_mid_gr_del` UNIQUE (`metalake_id`, `group_name`, `deleted_at`) - ) ENGINE=InnoDB; - -CREATE TABLE IF NOT EXISTS `group_role_rel` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id', - `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation deleted at', - PRIMARY KEY (`id`), - CONSTRAINT `uk_gi_ri_del` UNIQUE (`group_id`, `role_id`, `deleted_at`), - KEY `idx_gid` (`group_id`) - ) ENGINE=InnoDB; \ No newline at end of file diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 769f99958..cc815f9ee 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -44,7 +44,7 @@ You can also specify filter parameters by setting configuration entries of the f | Configuration item | Description | Default value | Required | Since version | |---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------|--------------------------------------------------|---------------| -| `gravitino.entity.store` | Which storage implementation to use. Key-value pair storage and relational storage are currently supported, the default value is `kv`, and the optional value is `relational`. | `kv` | No | 0.1.0 | +| `gravitino.entity.store` | Which entity storage implementation to use. Only`relational` storage is currently supported. | `relational` | No | 0.1.0 | | `gravitino.entity.store.kv` | Detailed implementation of KV storage. `RocksDB` storage is currently supported, and the implementation is `RocksDBKvBackend`. | `RocksDBKvBackend` | No | 0.1.0 | | `gravitino.entity.store.kv.rocksdbPath` | The storage path for RocksDB storage implementation. It supports both absolute and relative path, if the value is a relative path, the final path is `${GRAVITINO_HOME}/${PATH_YOU_HAVA_SET}`, default value is `${GRAVITINO_HOME}/data/rocksdb` | `${GRAVITINO_HOME}/data/rocksdb` | No | 0.1.0 | | `gravitino.entity.serde` | The serialization/deserialization class used to support entity storage. `proto' is currently supported. | `proto` | No | 0.1.0 | diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 59d89981d..7fa802ef6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -124,6 +124,7 @@ jersey-test-framework-core = { group = "org.glassfish.jersey.test-framework", na jersey-test-framework-provider-jetty = { group = "org.glassfish.jersey.test-framework.providers", name = "jersey-test-framework-provider-jetty", version.ref = "jersey" } jodd-core = { group = "org.jodd", name = "jodd-core", version.ref = "jodd" } mockito-core = { group = "org.mockito", name = "mockito-core", version.ref = "mockito" } +mockito-inline = { group = "org.mockito", name = "mockito-inline", version.ref = "mockito" } hive2-metastore = { group = "org.apache.hive", name = "hive-metastore", version.ref = "hive2"} hive2-exec = { group = "org.apache.hive", name = "hive-exec", version.ref = "hive2"} hive2-common = { group = "org.apache.hive", name = "hive-common", version.ref = "hive2"} diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java index 8eacdc885..a763737e8 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java @@ -206,7 +206,7 @@ public class AbstractIT { LOG.info("Running Gravitino Server in {} mode", testMode); - if ("true".equals(System.getenv("jdbcBackend"))) { + if ("MySQL".equalsIgnoreCase(System.getenv("jdbcBackend"))) { // Start MySQL docker instance. META_DATA = TestDatabaseName.MYSQL_JDBC_BACKEND; containerSuite.startMySQLContainer(META_DATA); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/store/relational/service/FilesetMetaServiceIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/store/relational/service/FilesetMetaServiceIT.java index 95c8834e1..1222cbd19 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/store/relational/service/FilesetMetaServiceIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/store/relational/service/FilesetMetaServiceIT.java @@ -76,7 +76,7 @@ public class FilesetMetaServiceIT { @BeforeAll public static void setup() { - Assumptions.assumeTrue("true".equals(System.getenv("jdbcBackend"))); + Assumptions.assumeTrue("MySQL".equals(System.getenv("jdbcBackend"))); TestDatabaseName META_DATA = TestDatabaseName.MYSQL_JDBC_BACKEND; containerSuite.startMySQLContainer(META_DATA); MySQLContainer MYSQL_CONTAINER = containerSuite.getMySQLContainer(); @@ -123,7 +123,7 @@ public class FilesetMetaServiceIT { @Test public void testDeleteFilesetVersionsByRetentionCount() throws IOException { - Assumptions.assumeTrue("true".equals(System.getenv("jdbcBackend"))); + Assumptions.assumeTrue("MySQL".equals(System.getenv("jdbcBackend"))); IdGenerator idGenerator = new RandomIdGenerator(); AuditInfo auditInfo = AuditInfo.builder().withCreator("creator").withCreateTime(Instant.now()).build();