This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new fe4ba9ce8f [INLONG-10375][Manager] Add field mapping strategy for CLS,
StarRocks and Elasticsearch (#10376)
fe4ba9ce8f is described below
commit fe4ba9ce8f7ca228e6b0b9c41681656c3899bc56
Author: fuweng11 <[email protected]>
AuthorDate: Fri Jun 7 21:33:34 2024 +0800
[INLONG-10375][Manager] Add field mapping strategy for CLS, StarRocks and
Elasticsearch (#10376)
---
.../fieldtype/strategy/ClsFieldTypeStrategy.java | 40 ++++
.../strategy/ElasticsearchFieldTypeStrategy.java | 40 ++++
.../strategy/StarRocksFieldTypeStrategy.java | 40 ++++
.../src/main/resources/cls-field-type-mapping.yaml | 231 +++++++++++++++++++++
.../src/main/resources/es-field-type-mapping.yaml | 231 +++++++++++++++++++++
5 files changed, 582 insertions(+)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/ClsFieldTypeStrategy.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/ClsFieldTypeStrategy.java
new file mode 100644
index 0000000000..a062430be4
--- /dev/null
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/ClsFieldTypeStrategy.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.common.fieldtype.strategy;
+
+import org.apache.inlong.manager.common.consts.SinkType;
+import org.apache.inlong.manager.common.fieldtype.FieldTypeMappingReader;
+
+import org.springframework.stereotype.Service;
+
+/**
+ * The CLS field type mapping strategy
+ */
+@Service
+public class ClsFieldTypeStrategy extends DefaultFieldTypeStrategy {
+
+ public ClsFieldTypeStrategy() {
+ this.reader = new FieldTypeMappingReader(SinkType.CLS);
+ }
+
+ @Override
+ public Boolean accept(String type) {
+ return SinkType.CLS.equals(type);
+ }
+
+}
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/ElasticsearchFieldTypeStrategy.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/ElasticsearchFieldTypeStrategy.java
new file mode 100644
index 0000000000..2fbdde601c
--- /dev/null
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/ElasticsearchFieldTypeStrategy.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.common.fieldtype.strategy;
+
+import org.apache.inlong.manager.common.consts.SinkType;
+import org.apache.inlong.manager.common.fieldtype.FieldTypeMappingReader;
+
+import org.springframework.stereotype.Service;
+
+/**
+ * The Elasticsearch field type mapping strategy
+ */
+@Service
+public class ElasticsearchFieldTypeStrategy extends DefaultFieldTypeStrategy {
+
+ public ElasticsearchFieldTypeStrategy() {
+ this.reader = new FieldTypeMappingReader(SinkType.ES);
+ }
+
+ @Override
+ public Boolean accept(String type) {
+ return SinkType.ES.equals(type);
+ }
+
+}
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/StarRocksFieldTypeStrategy.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/StarRocksFieldTypeStrategy.java
new file mode 100644
index 0000000000..60efd5c5e9
--- /dev/null
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/StarRocksFieldTypeStrategy.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.common.fieldtype.strategy;
+
+import org.apache.inlong.manager.common.consts.DataNodeType;
+import org.apache.inlong.manager.common.fieldtype.FieldTypeMappingReader;
+
+import org.springframework.stereotype.Service;
+
+/**
+ * The StarRocks field type mapping strategy
+ */
+@Service
+public class StarRocksFieldTypeStrategy extends DefaultFieldTypeStrategy {
+
+ public StarRocksFieldTypeStrategy() {
+ this.reader = new FieldTypeMappingReader(DataNodeType.STARROCKS);
+ }
+
+ @Override
+ public Boolean accept(String type) {
+ return DataNodeType.STARROCKS.equals(type);
+ }
+
+}
diff --git
a/inlong-manager/manager-common/src/main/resources/cls-field-type-mapping.yaml
b/inlong-manager/manager-common/src/main/resources/cls-field-type-mapping.yaml
new file mode 100644
index 0000000000..e9419e8479
--- /dev/null
+++
b/inlong-manager/manager-common/src/main/resources/cls-field-type-mapping.yaml
@@ -0,0 +1,231 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+stream.type.to.target.type.converter:
+
+ - source.type: TINYINT
+ target.type: LONG
+
+ - source.type: SMALLINT
+ target.type: LONG
+
+ - source.type: TINYINT UNSIGNED
+ target.type: LONG
+
+ - source.type: TINYINT UNSIGNED ZEROFILL
+ target.type: LONG
+
+ - source.type: INT
+ target.type: LONG
+
+ - source.type: INTEGER
+ target.type: LONG
+
+ - source.type: YEAR
+ target.type: LONG
+
+ - source.type: SHORT
+ target.type: LONG
+
+ - source.type: MEDIUMINT
+ target.type: LONG
+
+ - source.type: SMALLINT UNSIGNED
+ target.type: LONG
+
+ - source.type: SMALLINT UNSIGNED ZEROFILL
+ target.type: LONG
+
+ - source.type: BIGINT
+ target.type: LONG
+
+ - source.type: INT UNSIGNED
+ target.type: LONG
+
+ - source.type: MEDIUMINT UNSIGNED
+ target.type: LONG
+
+ - source.type: MEDIUMINT UNSIGNED ZEROFILL
+ target.type: LONG
+
+ - source.type: INT UNSIGNED ZEROFILL
+ target.type: LONG
+
+ - source.type: BIGINT UNSIGNED
+ target.type: LONG
+
+ - source.type: LONG
+ target.type: LONG
+
+ - source.type: BIGINT UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: SERIAL
+ target.type: DOUBLE
+
+ - source.type: FLOAT
+ target.type: DOUBLE
+
+ - source.type: FLOAT UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: FLOAT UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: DOUBLE
+ target.type: DOUBLE
+
+ - source.type: DOUBLE UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: DOUBLE UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: DOUBLE PRECISION
+ target.type: DOUBLE
+
+ - source.type: DOUBLE PRECISION UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: REAL
+ target.type: DOUBLE
+
+ - source.type: REAL UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: REAL UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: NUMERIC
+ target.type: DOUBLE
+
+ - source.type: NUMERIC UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: NUMERIC UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: DECIMAL
+ target.type: DOUBLE
+
+ - source.type: DECIMAL UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: DECIMAL UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: FIXED
+ target.type: DOUBLE
+
+ - source.type: FIXED UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: FIXED UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: BOOLEAN
+ target.type: DOUBLE
+
+ - source.type: DATE
+ target.type: STRING
+
+ - source.type: TIME
+ target.type: STRING
+
+ - source.type: DATETIME
+ target.type: STRING
+
+ - source.type: TIMESTAMP
+ target.type: STRING
+
+ - source.type: CHAR
+ target.type: STRING
+
+ - source.type: JSON
+ target.type: STRING
+
+ - source.type: BIT
+ target.type: STRING
+
+ - source.type: VARCHAR
+ target.type: STRING
+
+ - source.type: TEXT
+ target.type: STRING
+
+ - source.type: BLOB
+ target.type: STRING
+
+ - source.type: TINYBLOB
+ target.type: STRING
+
+ - source.type: TINYTEXT
+ target.type: STRING
+
+ - source.type: MEDIUMBLOB
+ target.type: STRING
+
+ - source.type: MEDIUMTEXT
+ target.type: STRING
+
+ - source.type: LONGBLOB
+ target.type: STRING
+
+ - source.type: LONGTEXT
+ target.type: STRING
+
+ - source.type: VARBINARY
+ target.type: STRING
+
+ - source.type: GEOMETRY
+ target.type: STRING
+
+ - source.type: POINT
+ target.type: STRING
+
+ - source.type: LINESTRING
+ target.type: STRING
+
+ - source.type: POLYGON
+ target.type: STRING
+
+ - source.type: MULTIPOINT
+ target.type: STRING
+
+ - source.type: MULTILINESTRING
+ target.type: STRING
+
+ - source.type: MULTIPOLYGON
+ target.type: STRING
+
+ - source.type: GEOMETRYCOLLECTION
+ target.type: STRING
+
+ - source.type: ENUM
+ target.type: STRING
+
+ - source.type: STRING
+ target.type: STRING
+
+ - source.type: BINARY
+ target.type: STRING
+
+ - source.type: BYTE
+ target.type: STRING
\ No newline at end of file
diff --git
a/inlong-manager/manager-common/src/main/resources/es-field-type-mapping.yaml
b/inlong-manager/manager-common/src/main/resources/es-field-type-mapping.yaml
new file mode 100644
index 0000000000..bd437876ef
--- /dev/null
+++
b/inlong-manager/manager-common/src/main/resources/es-field-type-mapping.yaml
@@ -0,0 +1,231 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+stream.type.to.target.type.converter:
+
+ - source.type: TINYINT
+ target.type: LONG
+
+ - source.type: SMALLINT
+ target.type: LONG
+
+ - source.type: TINYINT UNSIGNED
+ target.type: LONG
+
+ - source.type: TINYINT UNSIGNED ZEROFILL
+ target.type: LONG
+
+ - source.type: INT
+ target.type: LONG
+
+ - source.type: INTEGER
+ target.type: LONG
+
+ - source.type: YEAR
+ target.type: LONG
+
+ - source.type: SHORT
+ target.type: LONG
+
+ - source.type: MEDIUMINT
+ target.type: LONG
+
+ - source.type: SMALLINT UNSIGNED
+ target.type: LONG
+
+ - source.type: SMALLINT UNSIGNED ZEROFILL
+ target.type: LONG
+
+ - source.type: BIGINT
+ target.type: LONG
+
+ - source.type: INT UNSIGNED
+ target.type: LONG
+
+ - source.type: MEDIUMINT UNSIGNED
+ target.type: LONG
+
+ - source.type: MEDIUMINT UNSIGNED ZEROFILL
+ target.type: LONG
+
+ - source.type: INT UNSIGNED ZEROFILL
+ target.type: LONG
+
+ - source.type: BIGINT UNSIGNED
+ target.type: LONG
+
+ - source.type: LONG
+ target.type: LONG
+
+ - source.type: BIGINT UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: SERIAL
+ target.type: DOUBLE
+
+ - source.type: FLOAT
+ target.type: DOUBLE
+
+ - source.type: FLOAT UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: FLOAT UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: DOUBLE
+ target.type: DOUBLE
+
+ - source.type: DOUBLE UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: DOUBLE UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: DOUBLE PRECISION
+ target.type: DOUBLE
+
+ - source.type: DOUBLE PRECISION UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: REAL
+ target.type: DOUBLE
+
+ - source.type: REAL UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: REAL UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: NUMERIC
+ target.type: DOUBLE
+
+ - source.type: NUMERIC UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: NUMERIC UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: DECIMAL
+ target.type: DOUBLE
+
+ - source.type: DECIMAL UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: DECIMAL UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: FIXED
+ target.type: DOUBLE
+
+ - source.type: FIXED UNSIGNED
+ target.type: DOUBLE
+
+ - source.type: FIXED UNSIGNED ZEROFILL
+ target.type: DOUBLE
+
+ - source.type: BOOLEAN
+ target.type: DOUBLE
+
+ - source.type: DATE
+ target.type: TEXT
+
+ - source.type: TIME
+ target.type: TEXT
+
+ - source.type: DATETIME
+ target.type: TEXT
+
+ - source.type: TIMESTAMP
+ target.type: TEXT
+
+ - source.type: CHAR
+ target.type: TEXT
+
+ - source.type: JSON
+ target.type: TEXT
+
+ - source.type: BIT
+ target.type: TEXT
+
+ - source.type: VARCHAR
+ target.type: TEXT
+
+ - source.type: TEXT
+ target.type: TEXT
+
+ - source.type: BLOB
+ target.type: TEXT
+
+ - source.type: TINYBLOB
+ target.type: TEXT
+
+ - source.type: TINYTEXT
+ target.type: TEXT
+
+ - source.type: MEDIUMBLOB
+ target.type: TEXT
+
+ - source.type: MEDIUMTEXT
+ target.type: TEXT
+
+ - source.type: LONGBLOB
+ target.type: TEXT
+
+ - source.type: LONGTEXT
+ target.type: TEXT
+
+ - source.type: VARBINARY
+ target.type: TEXT
+
+ - source.type: GEOMETRY
+ target.type: TEXT
+
+ - source.type: POINT
+ target.type: TEXT
+
+ - source.type: LINESTRING
+ target.type: TEXT
+
+ - source.type: POLYGON
+ target.type: TEXT
+
+ - source.type: MULTIPOINT
+ target.type: TEXT
+
+ - source.type: MULTILINESTRING
+ target.type: TEXT
+
+ - source.type: MULTIPOLYGON
+ target.type: TEXT
+
+ - source.type: GEOMETRYCOLLECTION
+ target.type: TEXT
+
+ - source.type: ENUM
+ target.type: TEXT
+
+ - source.type: STRING
+ target.type: TEXT
+
+ - source.type: BINARY
+ target.type: TEXT
+
+ - source.type: BYTE
+ target.type: TEXT
\ No newline at end of file
