This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 5d8667236b [INLONG-8413][Doc] Update the description about data
Ingestion and Synchronization (#8414)
5d8667236b is described below
commit 5d8667236b38501f0d49a5f698390285d4f0b605
Author: Charles Zhang <[email protected]>
AuthorDate: Tue Jul 4 18:38:14 2023 +0800
[INLONG-8413][Doc] Update the description about data Ingestion and
Synchronization (#8414)
---
README.md | 64 +++++++++++-----------
.../inlong/manager/common/enums/GroupMode.java | 4 +-
.../inlong/manager/pojo/group/InlongGroupInfo.java | 2 +-
.../manager/pojo/group/InlongGroupPageRequest.java | 2 +-
.../manager/pojo/group/InlongGroupRequest.java | 2 +-
.../resource/sort/DefaultSortConfigOperator.java | 6 +-
.../service/source/StreamSourceService.java | 2 +-
.../service/source/StreamSourceServiceImpl.java | 3 +-
.../main/resources/h2/apache_inlong_manager.sql | 2 +-
.../manager-web/sql/apache_inlong_manager.sql | 2 +-
inlong-manager/manager-web/sql/changes-1.8.0.sql | 2 +-
11 files changed, 46 insertions(+), 45 deletions(-)
diff --git a/README.md b/README.md
index a8abadbef7..ced3449496 100644
--- a/README.md
+++ b/README.md
@@ -72,38 +72,38 @@ You can use InLong in the following ways：
- Visualize your operations on [InLong
dashboard](https://inlong.apache.org/docs/next/user_guide/dashboard_usage).
## Supported Data Nodes (Updating)
-| Type | Name | Version |
Architecture |
-|--------------|-------------------|------------------------------|-----------------------|
-| Extract Node | Auto Push | None | Standard
|
-| | File | None | Standard
|
-| | Kafka | 2.x |
Lightweight, Standard |
-| | MongoDB | >= 3.6 |
Lightweight, Standard |
-| | MQTT | >= 3.1 | Standard
|
-| | MySQL | 5.6, 5.7, 8.0.x |
Lightweight, Standard |
-| | Oracle | 11,12,19 |
Lightweight |
-| | PostgreSQL | 9.6, 10, 11, 12 |
Lightweight, Standard |
-| | Pulsar | 2.8.x |
Lightweight |
-| | Redis | 2.6.x | Standard
|
-| | SQLServer | 2012, 2014, 2016, 2017, 2019 |
Lightweight, Standard |
-| Load Node | Auto Consumption | None | Standard
|
-| | ClickHouse | 20.7+ |
Lightweight, Standard |
-| | Elasticsearch | 6.x, 7.x |
Lightweight, Standard |
-| | Greenplum | 4.x, 5.x, 6.x |
Lightweight, Standard |
-| | HBase | 2.2.x |
Lightweight, Standard |
-| | HDFS | 2.x, 3.x |
Lightweight, Standard |
-| | Hive | 1.x, 2.x, 3.x |
Lightweight, Standard |
-| | Iceberg | 0.12.x |
Lightweight, Standard |
-| | Hudi | 0.12.x |
Lightweight, Standard |
-| | Kafka | 2.x |
Lightweight, Standard |
-| | MySQL | 5.6, 5.7, 8.0.x |
Lightweight, Standard |
-| | Oracle | 11, 12, 19 |
Lightweight, Standard |
-| | PostgreSQL | 9.6, 10, 11, 12 |
Lightweight, Standard |
-| | SQLServer | 2012, 2014, 2016, 2017, 2019 |
Lightweight, Standard |
-| | TDSQL-PostgreSQL | 10.17 |
Lightweight, Standard |
-| | Doris | >= 0.13 |
Lightweight, Standard |
-| | StarRocks | >= 2.0 |
Lightweight, Standard |
-| | Kudu | >= 1.12.0 |
Lightweight, Standard |
-| | Redis | >= 3.0 |
Lightweight, Standard |
+| Type | Name | Version |
+|--------------|-------------------|------------------------------|
+| Extract Node | Auto Push | None |
+| | File | None |
+| | Kafka | 2.x |
+| | MongoDB | >= 3.6 |
+| | MQTT | >= 3.1 |
+| | MySQL | 5.6, 5.7, 8.0.x |
+| | Oracle | 11,12,19 |
+| | PostgreSQL | 9.6, 10, 11, 12 |
+| | Pulsar | 2.8.x |
+| | Redis | 2.6.x |
+| | SQLServer | 2012, 2014, 2016, 2017, 2019 |
+| Load Node | Auto Consumption | None |
+| | ClickHouse | 20.7+ |
+| | Elasticsearch | 6.x, 7.x |
+| | Greenplum | 4.x, 5.x, 6.x |
+| | HBase | 2.2.x |
+| | HDFS | 2.x, 3.x |
+| | Hive | 1.x, 2.x, 3.x |
+| | Iceberg | 0.12.x |
+| | Hudi | 0.12.x |
+| | Kafka | 2.x |
+| | MySQL | 5.6, 5.7, 8.0.x |
+| | Oracle | 11, 12, 19 |
+| | PostgreSQL | 9.6, 10, 11, 12 |
+| | SQLServer | 2012, 2014, 2016, 2017, 2019 |
+| | TDSQL-PostgreSQL | 10.17 |
+| | Doris | >= 0.13 |
+| | StarRocks | >= 2.0 |
+| | Kudu | >= 1.12.0 |
+| | Redis | >= 3.0 |
## Build InLong
More detailed instructions can be found at [Quick
Start](https://inlong.apache.org/docs/next/quick_start/how_to_build) section in
the documentation.
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
index 6e5b358feb..f0db2353b6 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
@@ -25,13 +25,13 @@ import lombok.Getter;
public enum GroupMode {
/**
- * Standard group init with all components in Inlong Cluster
+ * Standard mode(include Data Ingestion and Synchronization): group init
with all components in InLong Cluster
* StreamSource -> Agent/SDK -> DataProxy -> MQ Cache -> Sort -> StreamSink
*/
STANDARD("standard"),
/**
- * DataSync group init with sort in Inlong Cluster
+ * DataSync mode(only Data Synchronization): group init only with sort in
InLong Cluster
* StreamSource -> Sort -> StreamSink
*/
DATASYNC("datasync");
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java
index 956cd214ad..978fe1b473 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupInfo.java
@@ -72,7 +72,7 @@ public abstract class InlongGroupInfo extends BaseInlongGroup
{
@ApiModelProperty(value = "Whether to enable create resource? 0: disable,
1: enable")
private Integer enableCreateResource = 0;
- @ApiModelProperty(value = "Standard mode: 0, DataSync mode: 1")
+ @ApiModelProperty(value = "Standard mode(include Data Ingestion and
Synchronization): 0, DataSync mode(only Data Synchronization): 1")
private Integer inlongGroupMode;
@ApiModelProperty(value = "Data report type, default is 0.\n"
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java
index bf5cb82982..83855aa582 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupPageRequest.java
@@ -58,7 +58,7 @@ public class InlongGroupPageRequest extends PageRequest {
@ApiModelProperty(value = "The inlong cluster tag list")
private List<String> clusterTagList;
- @ApiModelProperty(value = "Standard mode: 0, DataSync mode: 1")
+ @ApiModelProperty(value = "Standard mode(include Data Ingestion and
Synchronization): 0, DataSync mode(only Data Synchronization): 1")
private Integer inlongGroupMode;
@ApiModelProperty(value = "Current user", hidden = true)
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java
index ae1f6677ef..a9cc46fc8b 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/group/InlongGroupRequest.java
@@ -86,7 +86,7 @@ public abstract class InlongGroupRequest extends
BaseInlongGroup {
@Range(min = 0, max = 1, message = "default is 1, only supports [0:
disable, 1: enable]")
private Integer enableCreateResource = 1;
- @ApiModelProperty(value = "Standard mode: 0, DataSync mode: 1")
+ @ApiModelProperty(value = "Standard mode(include Data Ingestion and
Synchronization): 0, DataSync mode(only Data Synchronization): 1")
@Range(min = 0, max = 1, message = "default is 0, only supports [0:
Standard, 1: DataSync]")
private Integer inlongGroupMode = 0;
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java
index b2a38db3dc..f97ff09bf7 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java
@@ -138,8 +138,8 @@ public class DefaultSortConfigOperator implements
SortConfigOperator {
if
(InlongConstants.STANDARD_MODE.equals(groupInfo.getInlongGroupMode())) {
if (CollectionUtils.isNotEmpty(transformResponses)) {
relations =
NodeRelationUtils.createNodeRelations(inlongStream);
- // in standard mode, replace upstream source node and
transform input fields node
- // to MQ node (which is inlong stream id)
+ // in standard mode(include Data Ingestion and
Synchronization), replace upstream source node and
+ // transform input fields node to MQ node (which is InLong
stream id)
String mqNodeName = sources.get(0).getSourceName();
Set<String> nodeNameSet = getInputNodeNames(sources,
transformResponses);
adjustTransformField(transformResponses, nodeNameSet,
mqNodeName);
@@ -181,7 +181,7 @@ public class DefaultSortConfigOperator implements
SortConfigOperator {
/**
* Set origin node to mq node for transform fields if necessary.
*
- * In standard mode for InlongGroup, transform input node must either be
+ * In standard mode(include Data Ingestion and Synchronization) for
InlongGroup, transform input node must either be
* mq source node or transform node, otherwise replace it with mq node
name.
*/
private void adjustTransformField(List<TransformResponse> transforms,
Set<String> nodeNameSet, String mqNodeName) {
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java
index 1b2333fe80..1bcb9f9966 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceService.java
@@ -81,7 +81,7 @@ public interface StreamSourceService {
* Get the StreamSource Map by the inlong group info and inlong stream
info list.
* <p/>
* If the group mode is DATASYNC, means not using any MQ as a cached
source, then just get all related sources.
- * Otherwise, if the group mode is STANDARD, need get the cached MQ
sources.
+ * Otherwise, if the group mode is STANDARD(include Data Ingestion and
Synchronization), need get the cached MQ sources.
*
* @param groupInfo inlong group info
* @param streamInfos inlong stream info list
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
index ee3c4ad70e..c18aef1fe9 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
@@ -238,7 +238,8 @@ public class StreamSourceServiceImpl implements
StreamSourceService {
.collect(Collectors.groupingBy(StreamSource::getInlongStreamId, HashMap::new,
Collectors.toCollection(ArrayList::new)));
} else {
- // if the group mode is STANDARD, needs to get the cached MQ
sources
+ // if the group mode is STANDARD(include Data Ingestion and
Synchronization),
+ // InLong needs to get the cached MQ sources
StreamSourceOperator sourceOperator =
operatorFactory.getInstance(groupInfo.getMqType());
result = sourceOperator.getSourcesMap(groupInfo, streamInfos,
streamSources);
}
diff --git
a/inlong-manager/manager-test/src/main/resources/h2/apache_inlong_manager.sql
b/inlong-manager/manager-test/src/main/resources/h2/apache_inlong_manager.sql
index 1989ccfdea..0f4c8a76c0 100644
---
a/inlong-manager/manager-test/src/main/resources/h2/apache_inlong_manager.sql
+++
b/inlong-manager/manager-test/src/main/resources/h2/apache_inlong_manager.sql
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS `inlong_group`
`max_length` int(11) DEFAULT '10240' COMMENT
'The maximum length of a single piece of data, unit: Byte',
`enable_zookeeper` tinyint(1) DEFAULT '0' COMMENT
'Whether to enable the zookeeper, 0-disable, 1-enable',
`enable_create_resource` tinyint(1) DEFAULT '1' COMMENT
'Whether to enable create resource? 0-disable, 1-enable',
- `inlong_group_mode` tinyint(1) DEFAULT '0' COMMENT 'Inlong
group mode, Standard mode: 0, DataSync mode: 1',
+ `inlong_group_mode` tinyint(1) DEFAULT '0' COMMENT 'InLong
group mode, Standard mode(include Data Ingestion and Synchronization): 0,
DataSync mode(only Data Synchronization): 1',
`data_report_type` int(4) DEFAULT '0' COMMENT 'Data
report type. 0: report to DataProxy and respond when the DataProxy received
data. 1: report to DataProxy and respond after DataProxy sends data. 2: report
to MQ and respond when the MQ received data',
`inlong_cluster_tag` varchar(128) DEFAULT NULL COMMENT 'The
cluster tag, which links to inlong_cluster table',
`ext_params` mediumtext DEFAULT NULL COMMENT
'Extended params, will be saved as JSON string',
diff --git a/inlong-manager/manager-web/sql/apache_inlong_manager.sql
b/inlong-manager/manager-web/sql/apache_inlong_manager.sql
index c5c1e3a4f4..e131511bee 100644
--- a/inlong-manager/manager-web/sql/apache_inlong_manager.sql
+++ b/inlong-manager/manager-web/sql/apache_inlong_manager.sql
@@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS `inlong_group`
`max_length` int(11) DEFAULT '10240' COMMENT
'The maximum length of a single piece of data, unit: Byte',
`enable_zookeeper` tinyint(1) DEFAULT '0' COMMENT
'Whether to enable the zookeeper, 0-disable, 1-enable',
`enable_create_resource` tinyint(1) DEFAULT '1' COMMENT
'Whether to enable create resource? 0-disable, 1-enable',
- `inlong_group_mode` tinyint(1) DEFAULT '0' COMMENT 'Inlong
group mode, Standard mode: 0, DataSync mode: 1',
+ `inlong_group_mode` tinyint(1) DEFAULT '0' COMMENT 'InLong
group mode, Standard mode(include Data Ingestion and Synchronization): 0,
DataSync mode(only Data Synchronization): 1',
`data_report_type` int(4) DEFAULT '0' COMMENT 'Data
report type. 0: report to DataProxy and respond when the DataProxy received
data. 1: report to DataProxy and respond after DataProxy sends data. 2: report
to MQ and respond when the MQ received data',
`inlong_cluster_tag` varchar(128) DEFAULT NULL COMMENT 'The
cluster tag, which links to inlong_cluster table',
`ext_params` mediumtext DEFAULT NULL COMMENT
'Extended params, will be saved as JSON string',
diff --git a/inlong-manager/manager-web/sql/changes-1.8.0.sql
b/inlong-manager/manager-web/sql/changes-1.8.0.sql
index 50b5991d5c..0a33640d1e 100644
--- a/inlong-manager/manager-web/sql/changes-1.8.0.sql
+++ b/inlong-manager/manager-web/sql/changes-1.8.0.sql
@@ -24,7 +24,7 @@ SET FOREIGN_KEY_CHECKS = 0;
USE `apache_inlong_manager`;
ALTER TABLE inlong_group
- CHANGE lightweight inlong_group_mode tinyint(1) DEFAULT 0 NULL COMMENT
'Inlong group mode, Standard mode: 0, DataSync mode: 1';
+ CHANGE lightweight inlong_group_mode tinyint(1) DEFAULT 0 NULL COMMENT
'InLong group mode, Standard mode(include Data Ingestion and Synchronization):
0, DataSync mode(only Data Synchronization): 1';
-- To support multi-tenant management in InLong, see
https://github.com/apache/inlong/issues/7914
CREATE TABLE IF NOT EXISTS `inlong_tenant`
