This is an automated email from the ASF dual-hosted git repository.
zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 0e96faea1a4 Add MariaDB migration E2E (#30535)
0e96faea1a4 is described below
commit 0e96faea1a4eddfbfbbeed4e4780135982d3325e
Author: Xinze Guo <[email protected]>
AuthorDate: Tue Mar 19 16:35:01 2024 +0800
Add MariaDB migration E2E (#30535)
* Add mariadb E2E
* Update operation e2e
* Fix ci
* Reset default password
---
.github/workflows/e2e-operation.yml | 2 +-
.github/workflows/nightly-e2e-operation.yml | 2 +-
.../inventory/InventoryRecordsCountCalculator.java | 3 +-
.../check/datasource/MySQLDataSourceChecker.java | 3 +-
.../atomic/storage/StorageContainerFactory.java | 3 +
.../impl/StorageContainerConfigurationFactory.java | 3 +
.../MariaDBContainerConfigurationFactory.java | 62 ++++++++++++++++
.../atomic/storage/impl/MariaDBContainer.java | 82 ++++++++++++++++++++++
.../e2e/env/runtime/DataSourceEnvironment.java | 8 +++
.../init-sql/mariadb/00-init-authority.sql | 35 +++++++++
.../container/init-sql/mariadb/99-be-ready.sql | 34 +++++++++
test/e2e/fixture/pom.xml | 5 ++
test/e2e/operation/pipeline/pom.xml | 5 ++
.../general/MySQLMigrationGeneralE2EIT.java | 9 ++-
.../migration/general/RulesMigrationE2EIT.java | 7 +-
.../data/pipeline/env/PipelineE2EEnvironment.java | 8 +++
.../container/compose/NativeContainerComposer.java | 1 +
.../framework/helper/PipelineCaseHelper.java | 5 +-
.../src/test/resources/env/it-env.properties | 3 +
.../src/test/resources/env/mariadb/global.yaml | 44 ++++++++++++
.../src/test/resources/env/mysql/01-initdb.sql | 1 -
21 files changed, 313 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/e2e-operation.yml
b/.github/workflows/e2e-operation.yml
index 372ed5d1d74..2506a9fadbb 100644
--- a/.github/workflows/e2e-operation.yml
+++ b/.github/workflows/e2e-operation.yml
@@ -60,7 +60,7 @@ jobs:
fail-fast: false
matrix:
operation: [ transaction, pipeline, showprocesslist ]
- image: [ { type: "it.docker.mysql.version", version: "mysql:5.7" }, {
type: "it.docker.postgresql.version", version: "postgres:12-alpine" }, { type:
"it.docker.opengauss.version", version:
"enmotech/opengauss:2.1.0,enmotech/opengauss:3.1.0" } ]
+ image: [ { type: "it.docker.mysql.version", version: "mysql:5.7" }, {
type: "it.docker.postgresql.version", version: "postgres:12-alpine" }, { type:
"it.docker.opengauss.version", version:
"enmotech/opengauss:2.1.0,enmotech/opengauss:3.1.0" }, { type:
"it.docker.mariadb.version", version: "mariadb:11" } ]
exclude:
- operation: showprocesslist
image: { type: "it.docker.postgresql.version", version:
"postgres:12-alpine" }
diff --git a/.github/workflows/nightly-e2e-operation.yml
b/.github/workflows/nightly-e2e-operation.yml
index aaae928e135..a3f7c5235e2 100644
--- a/.github/workflows/nightly-e2e-operation.yml
+++ b/.github/workflows/nightly-e2e-operation.yml
@@ -44,7 +44,7 @@ jobs:
fail-fast: false
matrix:
operation: [ transaction, pipeline, showprocesslist ]
- image: [ { type: "it.docker.mysql.version", version:
"mysql:5.7,mysql:8.0" }, { type: "it.docker.postgresql.version", version:
"postgres:10-alpine,postgres:11-alpine,postgres:12-alpine,postgres:13-alpine,postgres:14-alpine"
}, { type: "it.docker.opengauss.version", version:
"enmotech/opengauss:2.1.0,enmotech/opengauss:3.0.0" } ]
+ image: [ { type: "it.docker.mysql.version", version:
"mysql:5.7,mysql:8.0" }, { type: "it.docker.postgresql.version", version:
"postgres:10-alpine,postgres:11-alpine,postgres:12-alpine,postgres:13-alpine,postgres:14-alpine"
}, { type: "it.docker.opengauss.version", version:
"enmotech/opengauss:2.1.0,enmotech/opengauss:3.0.0" }, { type:
"it.docker.mariadb.version", version: "mariadb:11" } ]
exclude:
- operation: showprocesslist
image: { type: "it.docker.postgresql.version", version:
"postgres:10-alpine,postgres:11-alpine,postgres:12-alpine,postgres:13-alpine,postgres:14-alpine"
}
diff --git
a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/InventoryRecordsCountCalculator.java
b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/InventoryRecordsCountCalculator.java
index e5e2971ab33..86c461ad110 100644
---
a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/InventoryRecordsCountCalculator.java
+++
b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/preparer/inventory/InventoryRecordsCountCalculator.java
@@ -25,6 +25,7 @@ import
org.apache.shardingsphere.data.pipeline.core.datasource.PipelineDataSourc
import
org.apache.shardingsphere.data.pipeline.core.sqlbuilder.sql.PipelinePrepareSQLBuilder;
import
org.apache.shardingsphere.data.pipeline.core.exception.job.SplitPipelineJobByUniqueKeyException;
import org.apache.shardingsphere.infra.database.core.type.DatabaseType;
+import
org.apache.shardingsphere.infra.database.mariadb.type.MariaDBDatabaseType;
import org.apache.shardingsphere.infra.database.mysql.type.MySQLDatabaseType;
import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader;
@@ -72,7 +73,7 @@ public final class InventoryRecordsCountCalculator {
try (
Connection connection = dataSource.getConnection();
PreparedStatement preparedStatement =
connection.prepareStatement(estimatedCountSQL)) {
- if (databaseType instanceof MySQLDatabaseType) {
+ if (databaseType instanceof MySQLDatabaseType || databaseType
instanceof MariaDBDatabaseType) {
preparedStatement.setString(1, connection.getCatalog());
}
try (ResultSet resultSet = preparedStatement.executeQuery()) {
diff --git
a/kernel/data-pipeline/dialect/mysql/src/main/java/org/apache/shardingsphere/data/pipeline/mysql/check/datasource/MySQLDataSourceChecker.java
b/kernel/data-pipeline/dialect/mysql/src/main/java/org/apache/shardingsphere/data/pipeline/mysql/check/datasource/MySQLDataSourceChecker.java
index 36132ba2769..acf445a48ad 100644
---
a/kernel/data-pipeline/dialect/mysql/src/main/java/org/apache/shardingsphere/data/pipeline/mysql/check/datasource/MySQLDataSourceChecker.java
+++
b/kernel/data-pipeline/dialect/mysql/src/main/java/org/apache/shardingsphere/data/pipeline/mysql/check/datasource/MySQLDataSourceChecker.java
@@ -41,7 +41,8 @@ public final class MySQLDataSourceChecker implements
DialectDataSourceChecker {
private static final String SHOW_GRANTS_SQL = "SHOW GRANTS";
- private static final String[][] REQUIRED_PRIVILEGES = {{"ALL PRIVILEGES",
"ON *.*"}, {"REPLICATION SLAVE", "REPLICATION CLIENT", "ON *.*"}};
+ // BINLOG MONITOR is a synonym for REPLICATION CLIENT for MariaDB
+ private static final String[][] REQUIRED_PRIVILEGES = {{"ALL PRIVILEGES",
"ON *.*"}, {"REPLICATION SLAVE", "REPLICATION CLIENT", "ON *.*"}, {"REPLICATION
SLAVE", "BINLOG MONITOR", "ON *.*"}};
private static final Map<String, String> REQUIRED_VARIABLES = new
HashMap<>(3, 1F);
diff --git
a/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/StorageContainerFactory.java
b/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/StorageContainerFactory.java
index b4019af1311..bd4f7f30f93 100644
---
a/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/StorageContainerFactory.java
+++
b/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/StorageContainerFactory.java
@@ -22,6 +22,7 @@ import lombok.NoArgsConstructor;
import org.apache.shardingsphere.infra.database.core.type.DatabaseType;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.config.StorageContainerConfiguration;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.impl.H2Container;
+import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.impl.MariaDBContainer;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.impl.MySQLContainer;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.impl.OpenGaussContainer;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.impl.PostgreSQLContainer;
@@ -52,6 +53,8 @@ public final class StorageContainerFactory {
return new OpenGaussContainer(storageContainerImage,
storageContainerConfig);
case "H2":
return new H2Container(storageContainerConfig);
+ case "MariaDB":
+ return new MariaDBContainer(storageContainerImage,
storageContainerConfig);
default:
throw new RuntimeException(String.format("Database `%s` is
unknown.", databaseType.getType()));
}
diff --git
a/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/config/impl/StorageContainerConfigurationFactory.java
b/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/config/impl/StorageContainerConfigurationFactory.java
index 9f31fe5de1c..5c405559c71 100644
---
a/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/config/impl/StorageContainerConfigurationFactory.java
+++
b/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/config/impl/StorageContainerConfigurationFactory.java
@@ -22,6 +22,7 @@ import lombok.NoArgsConstructor;
import org.apache.shardingsphere.infra.database.core.type.DatabaseType;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.config.StorageContainerConfiguration;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.config.impl.h2.H2ContainerConfigurationFactory;
+import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.config.impl.mariadb.MariaDBContainerConfigurationFactory;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.config.impl.mysql.MySQLContainerConfigurationFactory;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.config.impl.opengauss.OpenGaussContainerConfigurationFactory;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.config.impl.postgresql.PostgreSQLContainerConfigurationFactory;
@@ -73,6 +74,8 @@ public final class StorageContainerConfigurationFactory {
return OpenGaussContainerConfigurationFactory.newInstance();
case "H2":
return H2ContainerConfigurationFactory.newInstance();
+ case "MariaDB":
+ return MariaDBContainerConfigurationFactory.newInstance();
default:
throw new RuntimeException(String.format("Database `%s` is
unknown.", databaseType.getType()));
}
diff --git
a/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/config/impl/mariadb/MariaDBContainerConfigurationFactory.java
b/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/config/impl/mariadb/MariaDBContainerConfigurationFactory.java
new file mode 100644
index 00000000000..acffb8d42e7
--- /dev/null
+++
b/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/config/impl/mariadb/MariaDBContainerConfigurationFactory.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.config.impl.mariadb;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.config.StorageContainerConfiguration;
+import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.impl.MariaDBContainer;
+import
org.apache.shardingsphere.test.e2e.env.container.atomic.util.ContainerUtils;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * MariaDB container configuration factory.
+ */
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public final class MariaDBContainerConfigurationFactory {
+
+ /**
+ * Create new instance of MariaDB container configuration.
+ *
+ * @return created instance
+ */
+ public static StorageContainerConfiguration newInstance() {
+ return new StorageContainerConfiguration(getCommand(),
getContainerEnvironments(), getMountedResources(), Collections.emptyMap(),
Collections.emptyMap());
+ }
+
+ private static String getCommand() {
+ return "--server-id=" + ContainerUtils.generateMySQLServerId();
+ }
+
+ private static Map<String, String> getContainerEnvironments() {
+ Map<String, String> result = new HashMap<>(2, 1F);
+ result.put("LANG", "C.UTF-8");
+ result.put("MYSQL_RANDOM_ROOT_PASSWORD", "yes");
+ return result;
+ }
+
+ private static Map<String, String> getMountedResources() {
+ Map<String, String> result = new HashMap<>(2, 1F);
+ result.put("/env/mysql/mysql8/my.cnf",
MariaDBContainer.MARIADB_CONF_IN_CONTAINER);
+ result.put("/env/mysql/01-initdb.sql",
"/docker-entrypoint-initdb.d/01-initdb.sql");
+ return result;
+ }
+}
diff --git
a/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/impl/MariaDBContainer.java
b/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/impl/MariaDBContainer.java
new file mode 100644
index 00000000000..eb2af4a2017
--- /dev/null
+++
b/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/container/atomic/storage/impl/MariaDBContainer.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.test.e2e.env.container.atomic.storage.impl;
+
+import com.google.common.base.Strings;
+import org.apache.shardingsphere.infra.database.core.type.DatabaseType;
+import org.apache.shardingsphere.infra.database.mysql.type.MySQLDatabaseType;
+import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader;
+import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.DockerStorageContainer;
+import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.config.StorageContainerConfiguration;
+
+import java.util.Collection;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * MariaDB container.
+ */
+public final class MariaDBContainer extends DockerStorageContainer {
+
+ public static final int EXPOSED_PORT = 3306;
+
+ public static final String MARIADB_CONF_IN_CONTAINER =
"/etc/mysql/mariadb.cnf";
+
+ private final StorageContainerConfiguration storageContainerConfig;
+
+ public MariaDBContainer(final String containerImage, final
StorageContainerConfiguration storageContainerConfig) {
+ super(TypedSPILoader.getService(DatabaseType.class, "MariaDB"),
Strings.isNullOrEmpty(containerImage) ? "mariadb:11" : containerImage);
+ this.storageContainerConfig = storageContainerConfig;
+ }
+
+ @Override
+ protected void configure() {
+ setCommands(storageContainerConfig.getContainerCommand());
+ addEnvs(storageContainerConfig.getContainerEnvironments());
+ mapResources(storageContainerConfig.getMountedResources());
+ super.configure();
+ }
+
+ @Override
+ protected Collection<String> getDatabaseNames() {
+ return storageContainerConfig.getDatabaseTypes().entrySet().stream()
+ .filter(entry -> entry.getValue() instanceof
MySQLDatabaseType).map(Entry::getKey).collect(Collectors.toList());
+ }
+
+ @Override
+ protected Collection<String> getExpectedDatabaseNames() {
+ return
storageContainerConfig.getExpectedDatabaseTypes().entrySet().stream()
+ .filter(entry -> entry.getValue() instanceof
MySQLDatabaseType).map(Entry::getKey).collect(Collectors.toList());
+ }
+
+ @Override
+ public int getExposedPort() {
+ return EXPOSED_PORT;
+ }
+
+ @Override
+ public int getMappedPort() {
+ return getMappedPort(EXPOSED_PORT);
+ }
+
+ @Override
+ protected Optional<String> getDefaultDatabaseName() {
+ return Optional.empty();
+ }
+}
diff --git
a/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/runtime/DataSourceEnvironment.java
b/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/runtime/DataSourceEnvironment.java
index d9cb57796c2..5cba7a630ab 100644
---
a/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/runtime/DataSourceEnvironment.java
+++
b/test/e2e/env/src/test/java/org/apache/shardingsphere/test/e2e/env/runtime/DataSourceEnvironment.java
@@ -37,6 +37,8 @@ public final class DataSourceEnvironment {
return "org.h2.Driver";
case "MySQL":
return "com.mysql.cj.jdbc.Driver";
+ case "MariaDB":
+ return "org.mariadb.jdbc.Driver";
case "PostgreSQL":
return "org.postgresql.Driver";
case "openGauss":
@@ -62,6 +64,9 @@ public final class DataSourceEnvironment {
case "MySQL":
return
String.format("jdbc:mysql://%s:%s?useSSL=true&requireSSL=true&enabledTLSProtocols=TLSv1.2,TLSv1.3&verifyServerCertificate=false"
+
"&useServerPrepStmts=true&useLocalSessionState=true&characterEncoding=utf-8&allowMultiQueries=true&rewriteBatchedStatements=true",
host, port);
+ case "MariaDB":
+ return
String.format("jdbc:mariadb://%s:%s?useSSL=false&useServerPrepStmts=true&useLocalSessionState=true&characterEncoding=utf-8&allowMultiQueries=true&rewriteBatchedStatements=true",
+ host, port);
case "PostgreSQL":
return
String.format("jdbc:postgresql://%s:%s/?ssl=on&sslmode=prefer", host, port);
case "openGauss":
@@ -90,6 +95,9 @@ public final class DataSourceEnvironment {
"jdbc:mysql://%s:%s/%s?useSSL=true&requireSSL=true&enabledTLSProtocols=TLSv1.2,TLSv1.3&verifyServerCertificate=false"
+
"&useServerPrepStmts=true&useLocalSessionState=true&characterEncoding=utf-8&allowPublicKeyRetrieval=true&allowMultiQueries=true&rewriteBatchedStatements=true",
host, port, dataSourceName);
+ case "MariaDB":
+ return
String.format("jdbc:mariadb://%s:%s/%s?useSSL=false&useServerPrepStmts=true&useLocalSessionState=true&characterEncoding=utf-8&allowMultiQueries=true"
+ + "&rewriteBatchedStatements=true", host, port,
dataSourceName);
case "PostgreSQL":
return
String.format("jdbc:postgresql://%s:%s/%s?ssl=on&sslmode=prefer", host, port,
dataSourceName);
case "openGauss":
diff --git
a/test/e2e/env/src/test/resources/container/init-sql/mariadb/00-init-authority.sql
b/test/e2e/env/src/test/resources/container/init-sql/mariadb/00-init-authority.sql
new file mode 100644
index 00000000000..0a41dba5edd
--- /dev/null
+++
b/test/e2e/env/src/test/resources/container/init-sql/mariadb/00-init-authority.sql
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+CREATE USER IF NOT EXISTS 'test_user'@'%' IDENTIFIED BY 'Test@123';
+GRANT ALL ON *.* TO 'test_user'@'%';
diff --git
a/test/e2e/env/src/test/resources/container/init-sql/mariadb/99-be-ready.sql
b/test/e2e/env/src/test/resources/container/init-sql/mariadb/99-be-ready.sql
new file mode 100644
index 00000000000..5ba1d9f9531
--- /dev/null
+++ b/test/e2e/env/src/test/resources/container/init-sql/mariadb/99-be-ready.sql
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+CREATE USER IF NOT EXISTS 'ready_user'@'%' IDENTIFIED BY 'Ready@123';
diff --git a/test/e2e/fixture/pom.xml b/test/e2e/fixture/pom.xml
index 48f3551e24f..24105665079 100644
--- a/test/e2e/fixture/pom.xml
+++ b/test/e2e/fixture/pom.xml
@@ -58,6 +58,11 @@
<artifactId>mysql-connector-j</artifactId>
<scope>runtime</scope>
</dependency>
+ <dependency>
+ <groupId>org.mariadb.jdbc</groupId>
+ <artifactId>mariadb-java-client</artifactId>
+ <scope>runtime</scope>
+ </dependency>
<dependency>
<groupId>org.opengauss</groupId>
<artifactId>opengauss-jdbc</artifactId>
diff --git a/test/e2e/operation/pipeline/pom.xml
b/test/e2e/operation/pipeline/pom.xml
index 17cfc4c101c..2ea66079391 100644
--- a/test/e2e/operation/pipeline/pom.xml
+++ b/test/e2e/operation/pipeline/pom.xml
@@ -81,6 +81,11 @@
<artifactId>mysql-connector-j</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.mariadb.jdbc</groupId>
+ <artifactId>mariadb-java-client</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
diff --git
a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/general/MySQLMigrationGeneralE2EIT.java
b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/general/MySQLMigrationGeneralE2EIT.java
index 80acb0ee519..4db002d461a 100644
---
a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/general/MySQLMigrationGeneralE2EIT.java
+++
b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/general/MySQLMigrationGeneralE2EIT.java
@@ -20,9 +20,9 @@ package
org.apache.shardingsphere.test.e2e.data.pipeline.cases.migration.general
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import
org.apache.shardingsphere.data.pipeline.scenario.migration.MigrationJobType;
+import
org.apache.shardingsphere.infra.algorithm.keygen.snowflake.SnowflakeKeyGenerateAlgorithm;
import org.apache.shardingsphere.infra.database.core.type.DatabaseType;
import org.apache.shardingsphere.infra.spi.type.typed.TypedSPILoader;
-import
org.apache.shardingsphere.infra.algorithm.keygen.snowflake.SnowflakeKeyGenerateAlgorithm;
import
org.apache.shardingsphere.test.e2e.data.pipeline.cases.PipelineContainerComposer;
import
org.apache.shardingsphere.test.e2e.data.pipeline.cases.migration.AbstractMigrationE2EIT;
import
org.apache.shardingsphere.test.e2e.data.pipeline.cases.task.E2EIncrementalTask;
@@ -50,7 +50,10 @@ import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
-@PipelineE2ESettings(database = @PipelineE2EDatabaseSettings(type = "MySQL",
scenarioFiles = "env/scenario/general/mysql.xml"))
+@PipelineE2ESettings(database = {
+ @PipelineE2EDatabaseSettings(type = "MySQL", scenarioFiles =
"env/scenario/general/mysql.xml"),
+ @PipelineE2EDatabaseSettings(type = "MariaDB", scenarioFiles =
"env/scenario/general/mysql.xml")
+})
@Slf4j
class MySQLMigrationGeneralE2EIT extends AbstractMigrationE2EIT {
@@ -115,6 +118,6 @@ class MySQLMigrationGeneralE2EIT extends
AbstractMigrationE2EIT {
}
private static boolean isEnabled() {
- return
PipelineE2ECondition.isEnabled(TypedSPILoader.getService(DatabaseType.class,
"MySQL"));
+ return
PipelineE2ECondition.isEnabled(TypedSPILoader.getService(DatabaseType.class,
"MySQL"), TypedSPILoader.getService(DatabaseType.class, "MariaDB"));
}
}
diff --git
a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/general/RulesMigrationE2EIT.java
b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/general/RulesMigrationE2EIT.java
index 8d0d0527083..5771efacf51 100644
---
a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/general/RulesMigrationE2EIT.java
+++
b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/general/RulesMigrationE2EIT.java
@@ -44,7 +44,10 @@ import static org.hamcrest.Matchers.is;
* 1) no any rule.
* 2) only encrypt rule.
*/
-@PipelineE2ESettings(fetchSingle = true, database =
@PipelineE2EDatabaseSettings(type = "MySQL", scenarioFiles =
"env/scenario/primary_key/text_primary_key/mysql.xml"))
+@PipelineE2ESettings(fetchSingle = true, database = {
+ @PipelineE2EDatabaseSettings(type = "MySQL", scenarioFiles =
"env/scenario/primary_key/text_primary_key/mysql.xml"),
+ @PipelineE2EDatabaseSettings(type = "MariaDB", scenarioFiles =
"env/scenario/primary_key/text_primary_key/mysql.xml")
+})
class RulesMigrationE2EIT extends AbstractMigrationE2EIT {
private static final String SOURCE_TABLE_NAME = "t_order";
@@ -93,6 +96,6 @@ class RulesMigrationE2EIT extends AbstractMigrationE2EIT {
}
private static boolean isEnabled() {
- return
PipelineE2ECondition.isEnabled(TypedSPILoader.getService(DatabaseType.class,
"MySQL"));
+ return
PipelineE2ECondition.isEnabled(TypedSPILoader.getService(DatabaseType.class,
"MySQL"), TypedSPILoader.getService(DatabaseType.class, "MariaDB"));
}
}
diff --git
a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/env/PipelineE2EEnvironment.java
b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/env/PipelineE2EEnvironment.java
index d068bac0c20..102f9a2dc58 100644
---
a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/env/PipelineE2EEnvironment.java
+++
b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/env/PipelineE2EEnvironment.java
@@ -22,6 +22,7 @@ import lombok.Getter;
import lombok.SneakyThrows;
import org.apache.shardingsphere.infra.database.core.type.DatabaseType;
import
org.apache.shardingsphere.test.e2e.data.pipeline.env.enums.PipelineEnvTypeEnum;
+import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.impl.MariaDBContainer;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.impl.MySQLContainer;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.impl.OpenGaussContainer;
import
org.apache.shardingsphere.test.e2e.env.container.atomic.storage.impl.PostgreSQLContainer;
@@ -45,6 +46,8 @@ public final class PipelineE2EEnvironment {
private final List<String> mysqlVersions;
+ private final List<String> mariadbVersions;
+
private final List<String> postgresqlVersions;
private final List<String> openGaussVersions;
@@ -55,6 +58,7 @@ public final class PipelineE2EEnvironment {
props = loadProperties();
itEnvType =
PipelineEnvTypeEnum.valueOf(props.getProperty("pipeline.it.env.type",
PipelineEnvTypeEnum.NONE.name()).toUpperCase());
mysqlVersions =
Arrays.stream(props.getOrDefault("pipeline.it.docker.mysql.version",
"").toString().split(",")).filter(each ->
!Strings.isNullOrEmpty(each)).collect(Collectors.toList());
+ mariadbVersions =
Arrays.stream(props.getOrDefault("pipeline.it.docker.mariadb.version",
"").toString().split(",")).filter(each ->
!Strings.isNullOrEmpty(each)).collect(Collectors.toList());
postgresqlVersions =
Arrays.stream(props.getOrDefault("pipeline.it.docker.postgresql.version",
"").toString().split(",")).filter(cs ->
!Strings.isNullOrEmpty(cs)).collect(Collectors.toList());
openGaussVersions =
Arrays.stream(props.getOrDefault("pipeline.it.docker.opengauss.version",
"").toString().split(",")).filter(cs ->
!Strings.isNullOrEmpty(cs)).collect(Collectors.toList());
oracleVersions =
Arrays.stream(props.getOrDefault("pipeline.it.docker.oracle.version",
"").toString().split(",")).filter(cs ->
!Strings.isNullOrEmpty(cs)).collect(Collectors.toList());
@@ -83,6 +87,8 @@ public final class PipelineE2EEnvironment {
switch (databaseType.getType()) {
case "MySQL":
return
Integer.parseInt(props.getOrDefault("pipeline.it.native.mysql.port",
MySQLContainer.MYSQL_EXPOSED_PORT).toString());
+ case "MariaDB":
+ return
Integer.parseInt(props.getOrDefault("pipeline.it.native.mariadb.port",
MariaDBContainer.EXPOSED_PORT).toString());
case "PostgreSQL":
return
Integer.parseInt(props.getOrDefault("pipeline.it.native.postgresql.port",
PostgreSQLContainer.POSTGRESQL_EXPOSED_PORT).toString());
case "openGauss":
@@ -147,6 +153,8 @@ public final class PipelineE2EEnvironment {
switch (databaseType.getType()) {
case "MySQL":
return mysqlVersions;
+ case "MariaDB":
+ return mariadbVersions;
case "PostgreSQL":
return postgresqlVersions;
case "openGauss":
diff --git
a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/container/compose/NativeContainerComposer.java
b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/container/compose/NativeContainerComposer.java
index 10a89e4cebf..b7e0e28b269 100644
---
a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/container/compose/NativeContainerComposer.java
+++
b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/container/compose/NativeContainerComposer.java
@@ -57,6 +57,7 @@ public final class NativeContainerComposer extends
BaseContainerComposer {
String jdbcUrl;
switch (databaseType.getType()) {
case "MySQL":
+ case "MariaDB":
String queryAllTables = String.format("select table_name from
information_schema.tables where table_schema='%s' and table_type='BASE TABLE'",
databaseName);
jdbcUrl = DataSourceEnvironment.getURL(databaseType,
"localhost", actualDatabasePort, databaseName);
try (
diff --git
a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/helper/PipelineCaseHelper.java
b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/helper/PipelineCaseHelper.java
index fe9ba1420ea..67aa8d1c518 100644
---
a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/helper/PipelineCaseHelper.java
+++
b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/framework/helper/PipelineCaseHelper.java
@@ -25,6 +25,7 @@ import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.shardingsphere.infra.database.core.type.DatabaseType;
+import
org.apache.shardingsphere.infra.database.mariadb.type.MariaDBDatabaseType;
import org.apache.shardingsphere.infra.database.mysql.type.MySQLDatabaseType;
import
org.apache.shardingsphere.infra.database.opengauss.type.OpenGaussDatabaseType;
import
org.apache.shardingsphere.infra.database.postgresql.type.PostgreSQLDatabaseType;
@@ -83,7 +84,7 @@ public final class PipelineCaseHelper {
public static List<Object[]> generateOrderInsertData(final DatabaseType
databaseType, final KeyGenerateAlgorithm keyGenerateAlgorithm, final int
insertRows) {
List<Object[]> result = new ArrayList<>(insertRows);
String emojiText = "☠️x☺️x✋x☹️";
- if (databaseType instanceof MySQLDatabaseType) {
+ if (databaseType instanceof MySQLDatabaseType || databaseType
instanceof MariaDBDatabaseType) {
for (int i = 0; i < insertRows; i++) {
int randomInt = generateInt(-100, 100);
Object orderId =
keyGenerateAlgorithm.generateKeys(mock(AlgorithmSQLContext.class),
1).iterator().next();
@@ -120,7 +121,7 @@ public final class PipelineCaseHelper {
}
return result;
}
- throw new UnsupportedOperationException("now support generate %s
insert data");
+ throw new UnsupportedOperationException(String.format("Not support
generate %s insert data", databaseType.getType()));
}
private static int generateInt(final int min, final int max) {
diff --git
a/test/e2e/operation/pipeline/src/test/resources/env/it-env.properties
b/test/e2e/operation/pipeline/src/test/resources/env/it-env.properties
index 3bc6e99ffd6..b367ad83f8e 100644
--- a/test/e2e/operation/pipeline/src/test/resources/env/it-env.properties
+++ b/test/e2e/operation/pipeline/src/test/resources/env/it-env.properties
@@ -27,6 +27,9 @@ pipeline.it.native.database=
pipeline.it.native.mysql.username=root
pipeline.it.native.mysql.password=Root@123
pipeline.it.native.mysql.port=3306
+pipeline.it.native.mariadb.username=root
+pipeline.it.native.mariadb.password=Root@123
+pipeline.it.native.mariadb.port=3306
pipeline.it.native.postgresql.username=postgres
pipeline.it.native.postgresql.password=Root@123
pipeline.it.native.postgresql.port=5432
diff --git
a/test/e2e/operation/pipeline/src/test/resources/env/mariadb/global.yaml
b/test/e2e/operation/pipeline/src/test/resources/env/mariadb/global.yaml
new file mode 100644
index 00000000000..a0715313452
--- /dev/null
+++ b/test/e2e/operation/pipeline/src/test/resources/env/mariadb/global.yaml
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+mode:
+ type: Cluster
+ repository:
+ type: ZooKeeper
+ props:
+ namespace: it_db_mariadb
+ server-lists: zk.host:2181
+
+authority:
+ users:
+ - user: proxy
+ password: Proxy@123
+ privilege:
+ type: ALL_PERMITTED
+
+sqlFederation:
+ sqlFederationEnabled: true
+ executionPlanCache:
+ initialCapacity: 2000
+ maximumSize: 65535
+
+props:
+ system-schema-metadata-enabled: false
+ max-connections-size-per-query: 1
+ kernel-executor-size: 16 # Infinite by default.
+ proxy-frontend-flush-threshold: 128 # The default value is 128.
+ sql-show: false
diff --git
a/test/e2e/operation/pipeline/src/test/resources/env/mysql/01-initdb.sql
b/test/e2e/operation/pipeline/src/test/resources/env/mysql/01-initdb.sql
index 6e04bb35ec1..2c892234af3 100644
--- a/test/e2e/operation/pipeline/src/test/resources/env/mysql/01-initdb.sql
+++ b/test/e2e/operation/pipeline/src/test/resources/env/mysql/01-initdb.sql
@@ -35,4 +35,3 @@ GRANT CREATE, DROP, SELECT, INSERT, UPDATE, DELETE, INDEX ON
pipeline_it_1.* TO
GRANT CREATE, DROP, SELECT, INSERT, UPDATE, DELETE, INDEX ON pipeline_it_2.*
TO `test_user`@`%`;
GRANT CREATE, DROP, SELECT, INSERT, UPDATE, DELETE, INDEX ON pipeline_it_3.*
TO `test_user`@`%`;
GRANT CREATE, DROP, SELECT, INSERT, UPDATE, DELETE, INDEX ON pipeline_it_4.*
TO `test_user`@`%`;
-ALTER USER `test_user`@`%` IDENTIFIED WITH mysql_native_password by 'Test@123';
