EMsnap commented on a change in pull request #2725:
URL: https://github.com/apache/incubator-inlong/pull/2725#discussion_r815291870
##########
File path:
inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/JobConstants.java
##########
@@ -51,27 +51,27 @@
public static final String JOB_DIR_FILTER_PATH = "job.filejob.dir.path";
//Binlog job
- private static final String JOB_DATABASE_USER = "job.binlogjob.user";
- private static final String JOB_DATABASE_PASSWORD =
"job.binlogjob.password";
- private static final String JOB_DATABASE_HOSTNAME =
"job.binlogjob.hostname";
- private static final String JOB_DATABASE_WHITELIST =
"job.binlogjob.tableWhiteList";
- private static final String JOB_DATABASE_SERVER_TIME_ZONE =
"job.binlogjob.database.serverTimezone";
- private static final String JOB_DATABASE_STORE_OFFSET_INTERVAL_MS =
"offset.binlogjob.offset.flush.interval.ms";
- private static final String JOB_DATABASE_STORE_HISTORY_FILENAME =
"job.binlogjob.database.history.file.filename";
- private static final String JOB_DATABASE_SNAPSHOT_MODE =
"job.binlogjob.database.snapshot.mode";
- private static final String JOB_DATABASE_OFFSET =
"job.binlogjob.database.offset";
+ public static final String JOB_DATABASE_USER = "job.binlogjob.user";
+ public static final String JOB_DATABASE_PASSWORD =
"job.binlogjob.password";
+ public static final String JOB_DATABASE_HOSTNAME =
"job.binlogjob.hostname";
+ public static final String JOB_DATABASE_WHITELIST =
"job.binlogjob.tableWhiteList";
+ public static final String JOB_DATABASE_SERVER_TIME_ZONE =
"job.binlogjob.database.serverTimezone";
+ public static final String JOB_DATABASE_STORE_OFFSET_INTERVAL_MS =
"offset.binlogjob.offset.flush.interval.ms";
+ public static final String JOB_DATABASE_STORE_HISTORY_FILENAME =
"job.binlogjob.database.history.file.filename";
+ public static final String JOB_DATABASE_SNAPSHOT_MODE =
"job.binlogjob.database.snapshot.mode";
+ public static final String JOB_DATABASE_OFFSET =
"job.binlogjob.database.offset";
//Kafka job
- private static final String SOURCE_KAFKA_TOPIC = "job.kafkajob.topic";
- private static final String SOURCE_KAFKA_KEY_DESERIALIZER =
"job.kafkajob.key.deserializer";
- private static final String SOURCE_KAFKA_VALUE_DESERIALIZER =
"job.kafkajob.value.Deserializer";
- private static final String SOURCE_KAFKA_BOOTSTRAP_SERVERS =
"job.kafkajob.bootstrap.servers";
- private static final String SOURCE_KAFKA_GROUP_ID =
"job.kafkajob.group.Id";
- private static final String SOURCE_KAFKA_RECORD_SPEED =
"job.kafkajob.record.speed";
- private static final String SOURCE_KAFKA_BYTE_SPEED_LIMIT =
"job.kafkajob.byte.speed.limit";
- private static final String SOURCE_KAFKA_MIN_INTERVAL =
"job.kafkajob.min.interval";
- private static final String SOURCE_KAFKA_OFFSET = "job.kafkajob.offset";
- private static final String SOURCE_KAFKA_READ_TIMEOUT =
"job.kafkajob.read.timeout";
+ public static final String JOB_KAFKA_TOPIC = "job.kafkajob.topic";
+ public static final String JOB_KAFKA_BOOTSTRAP_SERVERS =
"job.kafkajob.bootstrap.servers";
+ public static final String JOB_KAFKA_GROUP_ID = "job.kafkajob.group.id";
+ public static final String JOB_KAFKA_RECORD_SPEED_LIMIT =
"job.kafkajob.recordspeed.limit";
+ public static final String JOB_KAFKA_BYTE_SPEED_LIMIT =
"job.kafkajob.bytespeed.limit";
Review comment:
It is recommended to use private when there's no need to use public
##########
File path:
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/KafkaSource.java
##########
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.sources;
+
+import com.alibaba.fastjson.JSON;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.inlong.agent.conf.JobProfile;
+import org.apache.inlong.agent.plugin.Reader;
+import org.apache.inlong.agent.plugin.Source;
+import org.apache.inlong.agent.plugin.metrics.SourceJmxMetric;
+import org.apache.inlong.agent.plugin.metrics.SourceMetrics;
+import org.apache.inlong.agent.plugin.metrics.SourcePrometheusMetrics;
+import org.apache.inlong.agent.plugin.sources.reader.KafkaReader;
+import org.apache.inlong.agent.utils.ConfigUtil;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.common.TopicPartition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import static
org.apache.inlong.agent.constant.JobConstants.DEFAULT_JOB_LINE_FILTER;
+import static org.apache.inlong.agent.constant.JobConstants.JOB_KAFKA_OFFSET;
+import static
org.apache.inlong.agent.constant.JobConstants.JOB_KAFKA_PARTITION_OFFSET_DELIMITER;
+import static
org.apache.inlong.agent.constant.JobConstants.JOB_LINE_FILTER_PATTERN;
+import static
org.apache.inlong.agent.constant.JobConstants.JOB_OFFSET_DELIMITER;
+
+public class KafkaSource implements Source {
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(KafkaSource.class);
+
+ private static final String KAFKA_SOURCE_TAG_NAME =
"AgentKafkaSourceMetric";
+ private static final String JOB_KAFKAJOB_PARAM_PREFIX = "job.kafkajob.";
+ private static final String JOB_KAFKAJOB_TOPIC = "job.kafkajob.topic";
+ private static final String JOB_KAFKAJOB_BOOTSTRAP_SERVERS =
"job.kafkajob.bootstrap.servers";
+ private static final String JOB_KAFKAJOB_GROUP_ID =
"job.kafkajob.group.id";
+ private static final String JOB_KAFKAJOB_WAIT_TIMEOUT =
"job.kafkajob.wait.timeout";
+ //private static final String JOB_KAFKAJOB_PARTITION_OFFSET =
"job.kafkajob.topic.partition.offset";
+ private static final String KAFKA_COMMIT_AUTO = "enable.auto.commit";
+ private static final String KAFKA_DESERIALIZER_METHOD =
"org.apache.kafka.common.serialization.StringDeserializer";
+ private static final String KAFKA_KEY_DESERIALIZER = "key.deserializer";
+ private static final String KAFKA_VALUE_DESERIALIZER =
"value.deserializer";
+
+ private final SourceMetrics sourceMetrics;
+
+ public KafkaSource() {
+ if (ConfigUtil.isPrometheusEnabled()) {
+ this.sourceMetrics = new
SourcePrometheusMetrics(KAFKA_SOURCE_TAG_NAME);
Review comment:
add a atomic integer to define different source
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
