This is an automated email from the ASF dual-hosted git repository.
luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 452951d9f4 [INLONG-9312][Agent] Add data content style (#9313)
452951d9f4 is described below
commit 452951d9f43c3e28063cf3cefe7ebdab79863599
Author: justinwwhuang <[email protected]>
AuthorDate: Tue Nov 21 18:42:10 2023 +0800
[INLONG-9312][Agent] Add data content style (#9313)
* [INLONG-9312][Agent] Add data content style
* [INLONG-9312][Agent] Add data content style
---
.../src/main/java/org/apache/inlong/agent/constant/TaskConstants.java | 3 ++-
.../src/main/java/org/apache/inlong/agent/pojo/FileTask.java | 2 ++
.../src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java | 1 +
.../java/org/apache/inlong/agent/plugin/AgentBaseTestsHelper.java | 4 ++++
.../org/apache/inlong/agent/plugin/sources/TestLogFileSource.java | 1 -
5 files changed, 9 insertions(+), 2 deletions(-)
diff --git
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
index c501ec110b..37dd2d5da3 100755
---
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
+++
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
@@ -73,7 +73,8 @@ public class TaskConstants extends CommonConstants {
public static final String JOB_FILE_META_ENV_LIST = "job.fileTask.envList";
public static final String JOB_FILE_META_FILTER_BY_LABELS =
"job.fileTask.filterMetaByLabels";
public static final String JOB_FILE_PROPERTIES = "job.fileTask.properties";
- public static final String JOB_FILE_DATA_SOURCE_COLUMN_SEPARATOR =
"job.fileTask.dataSeparator";
+ public static final String SOURCE_DATA_CONTENT_STYLE =
"task.fileTask.dataContentStyle";
+ public static final String SOURCE_DATA_SEPARATOR =
"task.fileTask.dataSeparator";
public static final String JOB_FILE_MONITOR_INTERVAL =
"job.fileTask.monitorInterval";
public static final String JOB_FILE_MONITOR_STATUS =
"job.fileTask.monitorStatus";
public static final String JOB_FILE_MONITOR_EXPIRE =
"job.fileTask.monitorExpire";
diff --git
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileTask.java
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileTask.java
index ec8ce9f47f..f31ec703c2 100644
---
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileTask.java
+++
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileTask.java
@@ -48,6 +48,8 @@ public class FileTask {
// JSON string, the content format is List<Map<String, String>>
private String metaFields;
+ private String dataContentStyle;
+
private String dataSeparator;
// JSON string, the content format is Map<String,string>
diff --git
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
index 6d8cd16816..6ce6ba2d3d 100644
---
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
+++
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
@@ -138,6 +138,7 @@ public class TaskProfileDto {
fileTask.setDir(dir);
fileTask.setCollectType(taskConfig.getCollectType());
fileTask.setContentCollectType(taskConfig.getContentCollectType());
+ fileTask.setDataContentStyle(taskConfig.getDataContentStyle());
fileTask.setDataSeparator(taskConfig.getDataSeparator());
fileTask.setMaxFileCount(taskConfig.getMaxFileCount());
fileTask.setRetry(taskConfig.getRetry());
diff --git
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/AgentBaseTestsHelper.java
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/AgentBaseTestsHelper.java
index 2410c07ff3..465180cb8f 100755
---
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/AgentBaseTestsHelper.java
+++
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/AgentBaseTestsHelper.java
@@ -104,6 +104,10 @@ public class AgentBaseTestsHelper {
fileTaskConfig.setRetry(retry);
fileTaskConfig.setStartTime(startTime);
fileTaskConfig.setEndTime(endTime);
+ // mix: login|87601|968|67826|23579 or login|a=b&c=d&x=y&asdf
+ fileTaskConfig.setDataContentStyle("mix");
+ // 124 is the ASCII code of '|'
+ fileTaskConfig.setDataSeparator("124");
dataConfig.setExtParams(GSON.toJson(fileTaskConfig));
return dataConfig;
}
diff --git
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestLogFileSource.java
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestLogFileSource.java
index 67ce1dec76..b8e6d60fef 100644
---
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestLogFileSource.java
+++
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestLogFileSource.java
@@ -62,7 +62,6 @@ public class TestLogFileSource {
TaskProfile taskProfile = helper.getTaskProfile(1, pattern, false, 0L,
0L, TaskStateEnum.RUNNING);
instanceProfile = taskProfile.createInstanceProfile("",
fileName, "20230928", AgentUtils.getCurrentTime());
-
}
private LogFileSource getSource() {
