This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c3327b51b9 [Fix](Nereids) add nereids load function in read fields of
GlobalFunctionMgr and Database (#23248)
c3327b51b9 is described below
commit c3327b51b9eb69b6189e527e50dc55fb123a1a15
Author: mch_ucchi <[email protected]>
AuthorDate: Wed Aug 23 15:59:50 2023 +0800
[Fix](Nereids) add nereids load function in read fields of
GlobalFunctionMgr and Database (#23248)
add nereids load function in read fields of GlobalFunctionMgr and Database
to fix some udf is lost when restart FE and query with Nereids.
---
.../java/org/apache/doris/catalog/Database.java | 2 +-
.../org/apache/doris/catalog/FunctionRegistry.java | 1 -
.../org/apache/doris/catalog/FunctionUtil.java | 15 +++++++++++----
.../apache/doris/catalog/GlobalFunctionMgr.java | 2 +-
.../doris/nereids/trees/expressions/UdfTest.java | 22 ++++++++++++++++++++++
5 files changed, 35 insertions(+), 7 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 75250fb204..8da88fa146 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -634,7 +634,7 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
dbState = DbState.valueOf(Text.readString(in));
attachDbName = Text.readString(in);
- FunctionUtil.readFields(in, name2Function);
+ FunctionUtil.readFields(in, this.getFullName(), name2Function);
// read encryptKeys
if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_102) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java
index 53c729fd62..012f15faea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java
@@ -157,7 +157,6 @@ public class FunctionRegistry {
.collect(Collectors.joining(", ", "[", "]"));
}
-
public void addUdf(String dbName, String name, UdfBuilder builder) {
if (dbName == null) {
dbName = GLOBAL_FUNCTION;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java
index 403ae42b51..34f3a8b6f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionUtil.java
@@ -181,7 +181,8 @@ public class FunctionUtil {
}
}
- public static void readFields(DataInput in, ConcurrentMap<String,
ImmutableList<Function>> name2Function)
+ public static void readFields(DataInput in, String dbName,
+ ConcurrentMap<String, ImmutableList<Function>> name2Function)
throws IOException {
int numEntries = in.readInt();
for (int i = 0; i < numEntries; ++i) {
@@ -191,7 +192,11 @@ public class FunctionUtil {
for (int j = 0; j < numFunctions; ++j) {
builder.add(Function.read(in));
}
- name2Function.put(name, builder.build());
+ ImmutableList<Function> functions = builder.build();
+ name2Function.put(name, functions);
+ for (Function f : functions) {
+ translateToNereids(dbName, f);
+ }
}
}
@@ -234,7 +239,8 @@ public class FunctionUtil {
JavaUdaf.translateToNereidsFunction(dbName,
((AggregateFunction) function));
}
} catch (Exception e) {
- LOG.warn("Nereids create function {}:{} failed", dbName,
function.getFunctionName().getFunction(), e);
+ LOG.warn("Nereids create function {}:{} failed, caused by: {}",
dbName == null ? "_global_" : dbName,
+ function.getFunctionName().getFunction(), e);
}
return true;
}
@@ -246,7 +252,8 @@ public class FunctionUtil {
.collect(Collectors.toList());
Env.getCurrentEnv().getFunctionRegistry().dropUdf(dbName, fnName,
argTypes);
} catch (Exception e) {
- LOG.warn("Nereids drop function {}:{} failed", dbName,
function.getName(), e);
+ LOG.warn("Nereids drop function {}:{} failed, caused by: {}",
dbName == null ? "_global_" : dbName,
+ function.getName(), e);
}
return false;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/GlobalFunctionMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/GlobalFunctionMgr.java
index b62d0b2fe5..e95fc4b1bb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/GlobalFunctionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/GlobalFunctionMgr.java
@@ -57,7 +57,7 @@ public class GlobalFunctionMgr extends MetaObject {
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
- FunctionUtil.readFields(in, name2Function);
+ FunctionUtil.readFields(in, null, name2Function);
}
public synchronized void addFunction(Function function, boolean
ifNotExists) throws UserException {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java
index 35cb8ba14d..d6121f0588 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java
@@ -40,9 +40,15 @@ import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.nereids.util.PlanPatternMatchSupported;
import org.apache.doris.utframe.TestWithFeService;
+import com.google.common.collect.ImmutableList;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+
public class UdfTest extends TestWithFeService implements
PlanPatternMatchSupported {
@Override
protected void runBeforeAll() throws Exception {
@@ -171,4 +177,20 @@ public class UdfTest extends TestWithFeService implements
PlanPatternMatchSuppor
&&
relation.getProjects().get(0).child(0).equals(expected))
);
}
+
+ @Test
+ public void testReadFromStream() throws Exception {
+ createFunction("create global alias function f8(int) with parameter(n)
as hours_add(now(3), n)");
+ Env.getCurrentEnv().getFunctionRegistry().dropUdf(null, "f8",
+ ImmutableList.of(IntegerType.INSTANCE));
+
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ Env.getCurrentEnv().getGlobalFunctionMgr().write(new
DataOutputStream(outputStream));
+ byte[] buffer = outputStream.toByteArray();
+ ByteArrayInputStream inputStream = new ByteArrayInputStream(buffer);
+ Env.getCurrentEnv().getGlobalFunctionMgr().readFields(new
DataInputStream(inputStream));
+
+ Assertions.assertEquals(1, Env.getCurrentEnv().getFunctionRegistry()
+ .findUdfBuilder(connectContext.getDatabase(), "f8").size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
