This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ad8a8203a2 [fix](mysql compatibility) add an internal database mysql
to improve mysql compatibility (#22868)
ad8a8203a2 is described below
commit ad8a8203a2771ac81b3252586a884eca53d1094c
Author: hzq <[email protected]>
AuthorDate: Mon Aug 14 17:03:11 2023 +0800
[fix](mysql compatibility) add an internal database mysql to improve mysql
compatibility (#22868)
---
.../java/org/apache/doris/analysis/DropDbStmt.java | 6 +-
.../java/org/apache/doris/catalog/Database.java | 4 +
.../java/org/apache/doris/catalog/MysqlDb.java | 134 +++++++++++++++++++++
.../java/org/apache/doris/clone/TabletChecker.java | 2 +-
.../apache/doris/datasource/InternalCatalog.java | 31 +++--
.../org/apache/doris/httpv2/rest/ShowAction.java | 3 +-
.../master/PartitionInMemoryInfoCollector.java | 2 +-
.../apache/doris/mysql/privilege/RoleManager.java | 11 +-
.../transaction/DbUsedDataQuotaInfoCollector.java | 2 +-
.../java/org/apache/doris/catalog/MysqlDbTest.java | 39 ++++++
10 files changed, 214 insertions(+), 20 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java
index 86915b654b..2172477c29 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropDbStmt.java
@@ -19,6 +19,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.InfoSchemaDb;
+import org.apache.doris.catalog.MysqlDb;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
@@ -59,8 +60,9 @@ public class DropDbStmt extends DdlStmt {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME,
dbName);
}
dbName = ClusterNamespace.getFullName(getClusterName(), dbName);
- // Don't allowed to drop 'information_schema'
- if
(dbName.equalsIgnoreCase(ClusterNamespace.getFullName(getClusterName(),
InfoSchemaDb.DATABASE_NAME))) {
+ // Don't allowed to drop 'information_schema' & 'mysql'
+ if
(dbName.equalsIgnoreCase(ClusterNamespace.getFullName(getClusterName(),
InfoSchemaDb.DATABASE_NAME))
+ ||
dbName.equalsIgnoreCase(ClusterNamespace.getFullName(getClusterName(),
MysqlDb.DATABASE_NAME))) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_DBACCESS_DENIED_ERROR,
analyzer.getQualifiedUser(), dbName);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 54f1b6ce8b..a92992d530 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -762,6 +762,10 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
return
ClusterNamespace.getNameFromFullName(fullQualifiedName).equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME);
}
+ public boolean isMysqlDb() {
+ return
ClusterNamespace.getNameFromFullName(fullQualifiedName).equalsIgnoreCase(MysqlDb.DATABASE_NAME);
+ }
+
public synchronized void addEncryptKey(EncryptKey encryptKey, boolean
ifNotExists) throws UserException {
if (addEncryptKeyImpl(encryptKey, false, ifNotExists)) {
Env.getCurrentEnv().getEditLog().logAddEncryptKey(encryptKey);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlDb.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlDb.java
new file mode 100644
index 0000000000..9c91fd2d70
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlDb.java
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.alter.Alter;
+import org.apache.doris.analysis.AlterTableStmt;
+import org.apache.doris.analysis.CreateViewStmt;
+import org.apache.doris.cluster.ClusterNamespace;
+import org.apache.doris.common.Pair;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * This class is used for MySQL compatibility.
+ * The mysqldump requires this database to make some
+ * command arguments like --all-databases work as expected.
+ * Otherwise, commands like
+ * <p>
+ * mysqldump -u root -p --all-databases
+ * </p>
+ * will dump nothing.
+ * Native mysql has many system tables like slow_log under mysql database,
+ * but currently we do not create any tables under mysql database of doris.
+ * We will add useful system tables in the future.
+*/
+public class MysqlDb extends Database {
+ public static final String DATABASE_NAME = "mysql";
+ /**
+ * Database created by user will have database id starting from 10000
{@link Env#NEXT_ID_INIT_VALUE}.
+ * InfoSchemaDb takes id 0, so we assign id 1 to MysqlDb.
+ */
+ public static final long DATABASE_ID = 1L;
+
+ /**
+ * For test
+ */
+ public MysqlDb() {
+ super(DATABASE_ID, DATABASE_NAME);
+ initTables();
+ }
+
+ public MysqlDb(String cluster) {
+ super(DATABASE_ID, ClusterNamespace.getFullName(cluster,
DATABASE_NAME));
+ initTables();
+ }
+
+ /**
+ * Do nothing for now.
+ * If we need tables of mysql database in the future, create a MysqlTable
class like {@link SchemaTable}
+ */
+ private void initTables() {
+ }
+
+ /**
+ * This method must be re-implemented since {@link
Env#createView(CreateViewStmt)}
+ * will call this method. And create view should not succeed on this
database.
+ */
+ @Override
+ public Pair<Boolean, Boolean> createTableWithLock(Table table, boolean
isReplay, boolean setIfNotExist) {
+ return Pair.of(false, false);
+ }
+
+
+ /**
+ * Currently, rename a table of InfoSchemaDb will throw exception
+ * {@link Alter#processAlterTable(AlterTableStmt)}
+ * so we follow this design.
+ * @note: Rename a table of mysql database in MYSQL ls allowed.
+ */
+ @Override
+ public boolean createTable(Table table) {
+ return false;
+ }
+
+ @Override
+ public void dropTable(String name) {
+ // Do nothing.
+ }
+
+ /**
+ * MysqlDb is not persistent to bdb. It will be constructed everytime the
fe starts.
+ * {@link org.apache.doris.datasource.InternalCatalog#InternalCatalog()}
+ */
+ @Override
+ public void write(DataOutput out) throws IOException {
+ // Do nothing
+ }
+
+ /**
+ * Same with {@link InfoSchemaDb#readFields(DataInput)}
+ */
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ throw new IOException("Not support.");
+ }
+
+ /**
+ * Same with {@link InfoSchemaDb#getTableNullable(String)}
+ */
+ @Override
+ public Table getTableNullable(String name) {
+ return super.getTableNullable(name.toLowerCase());
+ }
+
+ public static boolean isMysqlDb(String dbName) {
+ if (dbName == null) {
+ return false;
+ }
+
+ String[] elements = dbName.split(ClusterNamespace.CLUSTER_DELIMITER);
+ String newDbName = dbName;
+ if (elements.length == 2) {
+ newDbName = elements[1];
+ }
+ return DATABASE_NAME.equalsIgnoreCase(newDbName);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
index 2b8bddbf9c..e7b762ef29 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
@@ -281,7 +281,7 @@ public class TabletChecker extends MasterDaemon {
continue;
}
- if (db.isInfoSchemaDb()) {
+ if (db.isInfoSchemaDb() || db.isMysqlDb()) {
continue;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index bedd5d4740..c70c9f0771 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -80,6 +80,7 @@ import org.apache.doris.catalog.MaterializedIndex.IndexState;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.MaterializedView;
import org.apache.doris.catalog.MetaIdGenerator.IdGeneratorBuffer;
+import org.apache.doris.catalog.MysqlDb;
import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.OlapTable;
@@ -215,12 +216,16 @@ public class InternalCatalog implements
CatalogIf<Database> {
private IcebergTableCreationRecordMgr icebergTableCreationRecordMgr = new
IcebergTableCreationRecordMgr();
public InternalCatalog() {
- // create info schema db
- final InfoSchemaDb db = new
InfoSchemaDb(SystemInfoService.DEFAULT_CLUSTER);
- db.setClusterName(SystemInfoService.DEFAULT_CLUSTER);
- // do not call unprotectedCreateDb, because it will cause loop
recursive when initializing Env singleton
- idToDb.put(db.getId(), db);
- fullNameToDb.put(db.getFullName(), db);
+ // create internal databases
+ List<Database> internalDbs = new ArrayList<>();
+ internalDbs.add(new InfoSchemaDb(SystemInfoService.DEFAULT_CLUSTER));
+ internalDbs.add(new MysqlDb(SystemInfoService.DEFAULT_CLUSTER));
+
+ for (Database idb : internalDbs) {
+ // do not call unprotectedCreateDb, because it will cause loop
recursive when initializing Env singleton
+ idToDb.put(idb.getId(), idb);
+ fullNameToDb.put(idb.getFullName(), idb);
+ }
}
@Override
@@ -1067,9 +1072,9 @@ public class InternalCatalog implements
CatalogIf<Database> {
String tableName = stmt.getTableName();
// check if db exists
- Database db = (Database) getDbOrDdlException(dbName);
- // InfoSchemaDb can not create table
- if (db instanceof InfoSchemaDb) {
+ Database db = getDbOrDdlException(dbName);
+ // InfoSchemaDb and MysqlDb can not create table manually
+ if (db instanceof InfoSchemaDb || db instanceof MysqlDb) {
ErrorReport.reportDdlException(ErrorCode.ERR_CANT_CREATE_TABLE,
tableName,
ErrorCode.ERR_CANT_CREATE_TABLE.getCode(), "not supported
create table in this database");
}
@@ -3074,15 +3079,15 @@ public class InternalCatalog implements
CatalogIf<Database> {
}
public long saveDb(CountingDataOutputStream dos, long checksum) throws
IOException {
- // 1 is for information_schema db, which does not need to be persisted.
- int dbCount = idToDb.size() - 1;
+ // 2 is for information_schema db & mysql db, which does not need to
be persisted.
+ int dbCount = idToDb.size() - 2;
checksum ^= dbCount;
dos.writeInt(dbCount);
for (Map.Entry<Long, Database> entry : idToDb.entrySet()) {
Database db = entry.getValue();
String dbName = db.getFullName();
- // Don't write information_schema db meta
- if (!InfoSchemaDb.isInfoSchemaDb(dbName)) {
+ // Don't write information_schema & mysql db meta
+ if (!InfoSchemaDb.isInfoSchemaDb(dbName) &&
!MysqlDb.isMysqlDb(dbName)) {
checksum ^= entry.getKey();
db.write(dos);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
index 8a0c7da4b2..fbece00f13 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
@@ -198,7 +198,8 @@ public class ShowAction extends RestBaseController {
} else {
for (long dbId : Env.getCurrentInternalCatalog().getDbIds()) {
DatabaseIf db =
Env.getCurrentInternalCatalog().getDbNullable(dbId);
- if (db == null || !(db instanceof Database) || ((Database)
db).isInfoSchemaDb()) {
+ if (db == null || !(db instanceof Database) || ((Database)
db).isInfoSchemaDb()
+ || ((Database) db).isMysqlDb()) {
continue;
}
totalSize += getDataSizeOfDatabase(db);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java
index 8bd02cb186..fefdf48b28 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java
@@ -56,7 +56,7 @@ public class PartitionInMemoryInfoCollector extends
MasterDaemon {
LOG.warn("Database [" + dbId + "] does not exist, skip to
update database used data quota");
continue;
}
- if (db.isInfoSchemaDb()) {
+ if (db.isInfoSchemaDb() || db.isMysqlDb()) {
continue;
}
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RoleManager.java
b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RoleManager.java
index 6d1a669c4f..285534d297 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RoleManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RoleManager.java
@@ -23,6 +23,7 @@ import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.analysis.WorkloadGroupPattern;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.InfoSchemaDb;
+import org.apache.doris.catalog.MysqlDb;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
@@ -189,13 +190,21 @@ public class RoleManager implements Writable {
if (roles.containsKey(userDefaultRoleName)) {
return roles.get(userDefaultRoleName);
}
- // grant read privs to database information_schema
+ // grant read privs to database information_schema & mysql
TablePattern tblPattern = new TablePattern(Auth.DEFAULT_CATALOG,
InfoSchemaDb.DATABASE_NAME, "*");
try {
tblPattern.analyze(SystemInfoService.DEFAULT_CLUSTER);
} catch (AnalysisException e) {
LOG.warn("should not happen", e);
}
+
+ tblPattern = new TablePattern(Auth.DEFAULT_CATALOG,
MysqlDb.DATABASE_NAME, "*");
+ try {
+ tblPattern.analyze(SystemInfoService.DEFAULT_CLUSTER);
+ } catch (AnalysisException e) {
+ LOG.warn("should not happen", e);
+ }
+
// grant read privs of default workload group
WorkloadGroupPattern workloadGroupPattern = new
WorkloadGroupPattern(WorkloadGroupMgr.DEFAULT_GROUP_NAME);
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java
index 25c7a3bd65..da6b16a1f3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java
@@ -50,7 +50,7 @@ public class DbUsedDataQuotaInfoCollector extends
MasterDaemon {
LOG.warn("Database [" + dbId + "] does not exist, skip to
update database used data quota");
continue;
}
- if (db.isInfoSchemaDb()) {
+ if (db.isInfoSchemaDb() || db.isMysqlDb()) {
continue;
}
try {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/MysqlDbTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/MysqlDbTest.java
new file mode 100644
index 0000000000..ebf7b661ea
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/MysqlDbTest.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.common.DdlException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class MysqlDbTest {
+ @Test
+ public void testNormal() throws IOException, DdlException {
+ Database db = new MysqlDb();
+
+ Assert.assertFalse(db.createTable(null));
+ Assert.assertFalse(db.createTableWithLock(null, false, false).first);
+ db.dropTable("authors");
+ db.write(null);
+ Assert.assertNull(db.getTableNullable("authors"));
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
