This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 16fdbbc0a9e [refactor](mysql compatibility) An abstract class for all
databases created for mysql compatibility (#23087) (#25109)
16fdbbc0a9e is described below
commit 16fdbbc0a9e8d1e96d2d20be0f6851489efc2e58
Author: zhiqqqq <[email protected]>
AuthorDate: Sun Oct 8 20:41:19 2023 -0500
[refactor](mysql compatibility) An abstract class for all databases created
for mysql compatibility (#23087) (#25109)
Better code structure for mysql compatibility databases.
---
.../java/org/apache/doris/catalog/Database.java | 16 ++--
.../org/apache/doris/catalog/InfoSchemaDb.java | 55 +------------
.../{MysqlDb.java => MysqlCompatibleDatabase.java} | 93 +++++++---------------
.../java/org/apache/doris/catalog/MysqlDb.java | 73 +----------------
.../java/org/apache/doris/clone/TabletChecker.java | 2 +-
.../apache/doris/datasource/InternalCatalog.java | 30 ++++---
.../org/apache/doris/httpv2/rest/ShowAction.java | 3 +-
.../master/PartitionInMemoryInfoCollector.java | 2 +-
.../transaction/DbUsedDataQuotaInfoCollector.java | 2 +-
.../org/apache/doris/catalog/InfoSchemaDbTest.java | 2 +-
.../java/org/apache/doris/catalog/MysqlDbTest.java | 2 +-
11 files changed, 69 insertions(+), 211 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 9728115dbf7..92100a98987 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -18,7 +18,6 @@
package org.apache.doris.catalog;
import org.apache.doris.catalog.TableIf.TableType;
-import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
@@ -764,14 +763,6 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
return FunctionUtil.getFunctions(name2Function);
}
- public boolean isInfoSchemaDb() {
- return
ClusterNamespace.getNameFromFullName(fullQualifiedName).equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME);
- }
-
- public boolean isMysqlDb() {
- return
ClusterNamespace.getNameFromFullName(fullQualifiedName).equalsIgnoreCase(MysqlDb.DATABASE_NAME);
- }
-
public synchronized void addEncryptKey(EncryptKey encryptKey, boolean
ifNotExists) throws UserException {
if (addEncryptKeyImpl(encryptKey, false, ifNotExists)) {
Env.getCurrentEnv().getEditLog().logAddEncryptKey(encryptKey);
@@ -914,4 +905,11 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
public String toString() {
return toJson();
}
+
+ // Return ture if database is created for mysql compatibility.
+ // Currently, we have two dbs that are created for this purpose,
InformationSchemaDb and MysqlDb,
+ public boolean isMysqlCompatibleDatabase() {
+ return false;
+ }
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/InfoSchemaDb.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/InfoSchemaDb.java
index 23a6bf34781..32143fd5898 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/InfoSchemaDb.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/InfoSchemaDb.java
@@ -18,76 +18,29 @@
package org.apache.doris.catalog;
import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.Pair;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
// Information schema used for MySQL compatible.
-public class InfoSchemaDb extends Database {
+public class InfoSchemaDb extends MysqlCompatibleDatabase {
public static final String DATABASE_NAME = "information_schema";
public static final long DATABASE_ID = 0L;
public InfoSchemaDb() {
super(DATABASE_ID, DATABASE_NAME);
- initTables();
}
public InfoSchemaDb(String cluster) {
super(DATABASE_ID, ClusterNamespace.getFullName(cluster,
DATABASE_NAME));
- initTables();
- }
-
- @Override
- public Pair<Boolean, Boolean> createTableWithLock(Table table, boolean
isReplay, boolean setIfNotExist) {
- return Pair.of(false, false);
- }
-
- @Override
- public boolean createTable(Table table) {
- // Do nothing.
- return false;
- }
-
- @Override
- public void dropTable(String name) {
- // Do nothing.
}
@Override
- public void write(DataOutput out) throws IOException {
- // Do nothing
- }
-
- public void readFields(DataInput in) throws IOException {
- throw new IOException("Not support.");
- }
-
- private void initTables() {
+ protected void initTables() {
for (Table table : SchemaTable.TABLE_MAP.values()) {
super.createTable(table);
}
}
@Override
- public Table getTableNullable(String name) {
- return super.getTableNullable(name.toLowerCase());
- }
-
- public static String getFullInfoSchemaDbName(String cluster) {
- return ClusterNamespace.getFullName(cluster, DATABASE_NAME);
- }
-
- public static boolean isInfoSchemaDb(String dbName) {
- if (dbName == null) {
- return false;
- }
- String[] ele = dbName.split(ClusterNamespace.CLUSTER_DELIMITER);
- String newDbName = dbName;
- if (ele.length == 2) {
- newDbName = ele[1];
- }
- return DATABASE_NAME.equalsIgnoreCase(newDbName);
+ public boolean createTable(Table table) {
+ return false;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlDb.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
similarity index 54%
copy from fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlDb.java
copy to
fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
index 9c91fd2d70c..1d4aa62dc9d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlDb.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlCompatibleDatabase.java
@@ -20,7 +20,6 @@ package org.apache.doris.catalog;
import org.apache.doris.alter.Alter;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.CreateViewStmt;
-import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.Pair;
import java.io.DataInput;
@@ -28,55 +27,21 @@ import java.io.DataOutput;
import java.io.IOException;
/**
- * This class is used for MySQL compatibility.
- * The mysqldump requires this database to make some
- * command arguments like --all-databases work as expected.
- * Otherwise, commands like
- * <p>
- * mysqldump -u root -p --all-databases
- * </p>
- * will dump nothing.
- * Native mysql has many system tables like slow_log under mysql database,
- * but currently we do not create any tables under mysql database of doris.
- * We will add useful system tables in the future.
-*/
-public class MysqlDb extends Database {
- public static final String DATABASE_NAME = "mysql";
- /**
- * Database created by user will have database id starting from 10000
{@link Env#NEXT_ID_INIT_VALUE}.
- * InfoSchemaDb takes id 0, so we assign id 1 to MysqlDb.
- */
- public static final long DATABASE_ID = 1L;
-
- /**
- * For test
- */
- public MysqlDb() {
- super(DATABASE_ID, DATABASE_NAME);
- initTables();
- }
+ * Abstract class for all databases created for mysql compatibility.
+ */
+public abstract class MysqlCompatibleDatabase extends Database {
+ public static int COUNT = 0;
- public MysqlDb(String cluster) {
- super(DATABASE_ID, ClusterNamespace.getFullName(cluster,
DATABASE_NAME));
+ public MysqlCompatibleDatabase(long id, String name) {
+ super(id, name);
initTables();
}
/**
- * Do nothing for now.
- * If we need tables of mysql database in the future, create a MysqlTable
class like {@link SchemaTable}
+ * Internal database is not persisted to bdb, it will be created when fe
starts.
+ * So driven class should implement this function to create table.
*/
- private void initTables() {
- }
-
- /**
- * This method must be re-implemented since {@link
Env#createView(CreateViewStmt)}
- * will call this method. And create view should not succeed on this
database.
- */
- @Override
- public Pair<Boolean, Boolean> createTableWithLock(Table table, boolean
isReplay, boolean setIfNotExist) {
- return Pair.of(false, false);
- }
-
+ protected abstract void initTables();
/**
* Currently, rename a table of InfoSchemaDb will throw exception
@@ -86,49 +51,51 @@ public class MysqlDb extends Database {
*/
@Override
public boolean createTable(Table table) {
- return false;
+ return super.createTable(table);
}
@Override
public void dropTable(String name) {
- // Do nothing.
+ // Do nothing
}
/**
- * MysqlDb is not persistent to bdb. It will be constructed everytime the
fe starts.
+ * MysqlCompatibleDatabase will not be persisted to bdb.
+ * It will be constructed everytime the fe starts. See
* {@link org.apache.doris.datasource.InternalCatalog#InternalCatalog()}
*/
@Override
public void write(DataOutput out) throws IOException {
- // Do nothing
+ throw new IOException("Not support");
}
/**
- * Same with {@link InfoSchemaDb#readFields(DataInput)}
+ * MysqlCompatibleDatabase should not be read from bdb.
*/
@Override
public void readFields(DataInput in) throws IOException {
throw new IOException("Not support.");
}
+ @Override
+ public boolean isMysqlCompatibleDatabase() {
+ return true;
+ }
+
/**
- * Same with {@link InfoSchemaDb#getTableNullable(String)}
+ * This method must be re-implemented since {@link
Env#createView(CreateViewStmt)}
+ * will call this method. And create view should not succeed under this
database.
*/
@Override
- public Table getTableNullable(String name) {
- return super.getTableNullable(name.toLowerCase());
+ public Pair<Boolean, Boolean> createTableWithLock(Table table, boolean
isReplay, boolean setIfNotExist) {
+ return Pair.of(false, false);
}
- public static boolean isMysqlDb(String dbName) {
- if (dbName == null) {
- return false;
- }
-
- String[] elements = dbName.split(ClusterNamespace.CLUSTER_DELIMITER);
- String newDbName = dbName;
- if (elements.length == 2) {
- newDbName = elements[1];
- }
- return DATABASE_NAME.equalsIgnoreCase(newDbName);
+ /**
+ * All tables of mysql compatible database has case-insensitive name
+ * */
+ @Override
+ public Table getTableNullable(String name) {
+ return super.getTableNullable(name.toLowerCase());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlDb.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlDb.java
index 9c91fd2d70c..5e6bd65507b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlDb.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlDb.java
@@ -17,15 +17,7 @@
package org.apache.doris.catalog;
-import org.apache.doris.alter.Alter;
-import org.apache.doris.analysis.AlterTableStmt;
-import org.apache.doris.analysis.CreateViewStmt;
import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.Pair;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
/**
* This class is used for MySQL compatibility.
@@ -40,7 +32,7 @@ import java.io.IOException;
* but currently we do not create any tables under mysql database of doris.
* We will add useful system tables in the future.
*/
-public class MysqlDb extends Database {
+public class MysqlDb extends MysqlCompatibleDatabase {
public static final String DATABASE_NAME = "mysql";
/**
* Database created by user will have database id starting from 10000
{@link Env#NEXT_ID_INIT_VALUE}.
@@ -53,82 +45,21 @@ public class MysqlDb extends Database {
*/
public MysqlDb() {
super(DATABASE_ID, DATABASE_NAME);
- initTables();
}
public MysqlDb(String cluster) {
super(DATABASE_ID, ClusterNamespace.getFullName(cluster,
DATABASE_NAME));
- initTables();
}
/**
* Do nothing for now.
* If we need tables of mysql database in the future, create a MysqlTable
class like {@link SchemaTable}
*/
- private void initTables() {
- }
-
- /**
- * This method must be re-implemented since {@link
Env#createView(CreateViewStmt)}
- * will call this method. And create view should not succeed on this
database.
- */
@Override
- public Pair<Boolean, Boolean> createTableWithLock(Table table, boolean
isReplay, boolean setIfNotExist) {
- return Pair.of(false, false);
- }
+ public void initTables() {}
-
- /**
- * Currently, rename a table of InfoSchemaDb will throw exception
- * {@link Alter#processAlterTable(AlterTableStmt)}
- * so we follow this design.
- * @note: Rename a table of mysql database in MYSQL ls allowed.
- */
@Override
public boolean createTable(Table table) {
return false;
}
-
- @Override
- public void dropTable(String name) {
- // Do nothing.
- }
-
- /**
- * MysqlDb is not persistent to bdb. It will be constructed everytime the
fe starts.
- * {@link org.apache.doris.datasource.InternalCatalog#InternalCatalog()}
- */
- @Override
- public void write(DataOutput out) throws IOException {
- // Do nothing
- }
-
- /**
- * Same with {@link InfoSchemaDb#readFields(DataInput)}
- */
- @Override
- public void readFields(DataInput in) throws IOException {
- throw new IOException("Not support.");
- }
-
- /**
- * Same with {@link InfoSchemaDb#getTableNullable(String)}
- */
- @Override
- public Table getTableNullable(String name) {
- return super.getTableNullable(name.toLowerCase());
- }
-
- public static boolean isMysqlDb(String dbName) {
- if (dbName == null) {
- return false;
- }
-
- String[] elements = dbName.split(ClusterNamespace.CLUSTER_DELIMITER);
- String newDbName = dbName;
- if (elements.length == 2) {
- newDbName = elements[1];
- }
- return DATABASE_NAME.equalsIgnoreCase(newDbName);
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
index e7b762ef295..09e61ec59b2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletChecker.java
@@ -281,7 +281,7 @@ public class TabletChecker extends MasterDaemon {
continue;
}
- if (db.isInfoSchemaDb() || db.isMysqlDb()) {
+ if (db.isMysqlCompatibleDatabase()) {
continue;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 9960d71b7dd..866d99db4a4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -80,6 +80,7 @@ import org.apache.doris.catalog.MaterializedIndex.IndexState;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.MaterializedView;
import org.apache.doris.catalog.MetaIdGenerator.IdGeneratorBuffer;
+import org.apache.doris.catalog.MysqlCompatibleDatabase;
import org.apache.doris.catalog.MysqlDb;
import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.OdbcTable;
@@ -184,7 +185,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
@@ -214,11 +214,12 @@ public class InternalCatalog implements
CatalogIf<Database> {
public InternalCatalog() {
// create internal databases
- List<Database> internalDbs = new ArrayList<>();
- internalDbs.add(new InfoSchemaDb(SystemInfoService.DEFAULT_CLUSTER));
- internalDbs.add(new MysqlDb(SystemInfoService.DEFAULT_CLUSTER));
+ List<MysqlCompatibleDatabase> mysqlCompatibleDatabases = new
ArrayList<>();
+ mysqlCompatibleDatabases.add(new
InfoSchemaDb(SystemInfoService.DEFAULT_CLUSTER));
+ mysqlCompatibleDatabases.add(new
MysqlDb(SystemInfoService.DEFAULT_CLUSTER));
+ MysqlCompatibleDatabase.COUNT = 2;
- for (Database idb : internalDbs) {
+ for (MysqlCompatibleDatabase idb : mysqlCompatibleDatabases) {
// do not call unprotectedCreateDb, because it will cause loop
recursive when initializing Env singleton
idToDb.put(idb.getId(), idb);
fullNameToDb.put(idb.getFullName(), idb);
@@ -856,6 +857,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
// check database
Database db = (Database) getDbOrDdlException(dbName);
+ if (db.isMysqlCompatibleDatabase()) {
+ throw new DdlException("Drop table from this database is not
allowed.");
+ }
+
db.writeLockOrDdlException();
try {
Table table = db.getTableNullable(tableName);
@@ -1074,7 +1079,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
// check if db exists
Database db = getDbOrDdlException(dbName);
// InfoSchemaDb and MysqlDb can not create table manually
- if (db instanceof InfoSchemaDb || db instanceof MysqlDb) {
+ if (db.isMysqlCompatibleDatabase()) {
ErrorReport.reportDdlException(ErrorCode.ERR_CANT_CREATE_TABLE,
tableName,
ErrorCode.ERR_CANT_CREATE_TABLE.getCode(), "not supported
create table in this database");
}
@@ -3108,14 +3113,19 @@ public class InternalCatalog implements
CatalogIf<Database> {
public long saveDb(CountingDataOutputStream dos, long checksum) throws
IOException {
// 2 is for information_schema db & mysql db, which does not need to
be persisted.
- int dbCount = idToDb.size() - 2;
+ // And internal database could not be dropped, so we assert dbCount >= 0
+ int dbCount = idToDb.size() - MysqlCompatibleDatabase.COUNT;
+ if (dbCount < 0) {
+ throw new IOException("Invalid database count");
+ }
+
checksum ^= dbCount;
dos.writeInt(dbCount);
+
for (Map.Entry<Long, Database> entry : idToDb.entrySet()) {
Database db = entry.getValue();
- String dbName = db.getFullName();
- // Don't write information_schema & mysql db meta
- if (!InfoSchemaDb.isInfoSchemaDb(dbName) &&
!MysqlDb.isMysqlDb(dbName)) {
+ // Don't write internal database meta.
+ if (!db.isMysqlCompatibleDatabase()) {
checksum ^= entry.getKey();
db.write(dos);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
index fbece00f13e..c1dc8f43802 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
@@ -198,8 +198,7 @@ public class ShowAction extends RestBaseController {
} else {
for (long dbId : Env.getCurrentInternalCatalog().getDbIds()) {
DatabaseIf db =
Env.getCurrentInternalCatalog().getDbNullable(dbId);
- if (db == null || !(db instanceof Database) || ((Database)
db).isInfoSchemaDb()
- || ((Database) db).isMysqlDb()) {
+ if (db == null || !(db instanceof Database) || ((Database)
db).isMysqlCompatibleDatabase()) {
continue;
}
totalSize += getDataSizeOfDatabase(db);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java
index fefdf48b282..22f2d41cfad 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/master/PartitionInMemoryInfoCollector.java
@@ -56,7 +56,7 @@ public class PartitionInMemoryInfoCollector extends
MasterDaemon {
LOG.warn("Database [" + dbId + "] does not exist, skip to
update database used data quota");
continue;
}
- if (db.isInfoSchemaDb() || db.isMysqlDb()) {
+ if (db.isMysqlCompatibleDatabase()) {
continue;
}
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java
index da6b16a1f3e..87d509bbb4d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java
@@ -50,7 +50,7 @@ public class DbUsedDataQuotaInfoCollector extends
MasterDaemon {
LOG.warn("Database [" + dbId + "] does not exist, skip to
update database used data quota");
continue;
}
- if (db.isInfoSchemaDb() || db.isMysqlDb()) {
+ if (db.isMysqlCompatibleDatabase()) {
continue;
}
try {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/InfoSchemaDbTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/InfoSchemaDbTest.java
index ca33ad0bba5..9abb1322e6d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/InfoSchemaDbTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/InfoSchemaDbTest.java
@@ -32,7 +32,7 @@ public class InfoSchemaDbTest {
Assert.assertFalse(db.createTable(null));
Assert.assertFalse(db.createTableWithLock(null, false, false).first);
db.dropTable("authors");
- db.write(null);
+ Assert.assertThrows(IOException.class, () -> db.write(null));
Assert.assertNull(db.getTableNullable("authors"));
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/MysqlDbTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/MysqlDbTest.java
index ebf7b661ea1..788780202b4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/MysqlDbTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/MysqlDbTest.java
@@ -32,7 +32,7 @@ public class MysqlDbTest {
Assert.assertFalse(db.createTable(null));
Assert.assertFalse(db.createTableWithLock(null, false, false).first);
db.dropTable("authors");
- db.write(null);
+ Assert.assertThrows(IOException.class, () -> db.write(null));
Assert.assertNull(db.getTableNullable("authors"));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
