This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f09f4a4564a [fix](logstash) fix group_commit error in logstash (#40450)
f09f4a4564a is described below
commit f09f4a4564a07405a6fbd37196107135a35b1e2f
Author: Kang <[email protected]>
AuthorDate: Fri Sep 6 18:37:11 2024 +0800
[fix](logstash) fix group_commit error in logstash (#40450)
Do not set label when group_commit is set in headers.
---
extension/logstash/lib/logstash/outputs/doris.rb | 12 ++++++++++--
extension/logstash/logstash-output-doris.gemspec | 2 +-
2 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/extension/logstash/lib/logstash/outputs/doris.rb
b/extension/logstash/lib/logstash/outputs/doris.rb
index 34124f446bb..ce0fe1df760 100644
--- a/extension/logstash/lib/logstash/outputs/doris.rb
+++ b/extension/logstash/lib/logstash/outputs/doris.rb
@@ -99,6 +99,12 @@ class LogStash::Outputs::Doris < LogStash::Outputs::Base
@request_headers = make_request_headers
@logger.info("request headers: ", @request_headers)
+ @group_commit = false
+ if http_headers.has_key?("group_commit") && http_headers["group_commit"]
!= "off_mode"
+ @group_commit = true
+ end
+ @logger.info("group_commit: ", @group_commit)
+
@init_time = Time.now.to_i # seconds
@total_bytes = java.util.concurrent.atomic.AtomicLong.new(0)
@total_rows = java.util.concurrent.atomic.AtomicLong.new(0)
@@ -188,9 +194,11 @@ class LogStash::Outputs::Doris < LogStash::Outputs::Base
hosts = get_host_addresses()
http_headers = @request_headers.dup
- http_headers["label"] = label_prefix + "_" + @db + "_" + @table + "_" +
Time.now.strftime('%Y%m%d_%H%M%S_%L_' + SecureRandom.uuid)
+ if !@group_commit
+ # only set label if group_commit is off_mode or not set, since lable
can not be used with group_commit
+ http_headers["label"] = label_prefix + "_" + @db + "_" + @table + "_"
+ Time.now.strftime('%Y%m%d_%H%M%S_%L_' + SecureRandom.uuid)
+ end
- # @request_headers["label"] = label_prefix + "_" + @db + "_" + @table +
"_" + Time.now.strftime('%Y%m%d%H%M%S_%L')
req_count = 0
sleep_for = 1
while true
diff --git a/extension/logstash/logstash-output-doris.gemspec
b/extension/logstash/logstash-output-doris.gemspec
index 163ba260f07..6536d89ccc3 100644
--- a/extension/logstash/logstash-output-doris.gemspec
+++ b/extension/logstash/logstash-output-doris.gemspec
@@ -18,7 +18,7 @@ under the License.
=end
Gem::Specification.new do |s|
s.name = 'logstash-output-doris'
- s.version = '1.0.0'
+ s.version = '1.0.1'
s.author = 'Apache Doris'
s.email = '[email protected]'
s.homepage = 'http://doris.apache.org'
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
