This is an automated email from the ASF dual-hosted git repository.

zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 17919f8aef [MINOR][VL] Bound Kafka consume waits by the remaining poll 
timeout (#13449)
17919f8aef is described below

commit 17919f8aefbda14dab15c04c68e57111eab644ed
Author: Reema <[email protected]>
AuthorDate: Thu Nov 12 16:44:47 2026 +0300

    [MINOR][VL] Bound Kafka consume waits by the remaining poll timeout (#13449)
---
 cpp/velox/operators/reader/KafkaDataSource.cc |  47 +++--
 cpp/velox/operators/reader/KafkaDataSource.h  |   7 +
 cpp/velox/tests/CMakeLists.txt                |   7 +-
 gluten-ut/native/KafkaPollTimeoutTest.cc      | 283 ++++++++++++++++++++++++++
 4 files changed, 331 insertions(+), 13 deletions(-)

diff --git a/cpp/velox/operators/reader/KafkaDataSource.cc 
b/cpp/velox/operators/reader/KafkaDataSource.cc
index 20c1ae575d..2aedf56504 100644
--- a/cpp/velox/operators/reader/KafkaDataSource.cc
+++ b/cpp/velox/operators/reader/KafkaDataSource.cc
@@ -194,14 +194,42 @@ std::optional<RowVectorPtr> 
KafkaDataSource::next(uint64_t size, ContinueFuture&
   return nullptr;
 }
 
+std::chrono::steady_clock::time_point KafkaDataSource::pollTime() const {
+  return std::chrono::steady_clock::now();
+}
+
+RdKafka::Message* KafkaDataSource::consume(int timeoutMs) {
+  return consumer_->consume(timeoutMs);
+}
+
 std::vector<std::unique_ptr<RdKafka::Message>> KafkaDataSource::poll(uint64_t 
maxRows) {
   std::vector<std::unique_ptr<RdKafka::Message>> messages;
   const auto pollTimeoutMs = split_->pollTimeoutMs > 0 ? split_->pollTimeoutMs 
: kDefaultPollTimeoutMs;
-  const auto deadline = std::chrono::steady_clock::now() + 
std::chrono::milliseconds(pollTimeoutMs);
+  const auto deadline = pollTime() + std::chrono::milliseconds(pollTimeoutMs);
+  const auto failOnTimeout = [&]() {
+    VELOX_FAIL(
+        "Timed out after {} ms fetching offset {} of {}-{}",
+        pollTimeoutMs,
+        nextOffset_,
+        split_->topic,
+        split_->partition);
+  };
+  // Bound record-consumption waits by the time left in this poll. Metadata 
and recovery
+  // operations retain their separate timeout allowances.
+  const auto remainingTimeoutMs = [this, &deadline]() {
+    return static_cast<int>(std::clamp<int64_t>(
+        std::chrono::duration_cast<std::chrono::milliseconds>(deadline - 
pollTime()).count(),
+        0,
+        std::numeric_limits<int>::max()));
+  };
 
   while (!splitDone_ && messages.size() < maxRows) {
+    const auto timeoutMs = remainingTimeoutMs();
+    if (timeoutMs == 0) {
+      break;
+    }
     // Only wait for more records while the batch is empty.
-    std::unique_ptr<RdKafka::Message> msg(consumer_->consume(messages.empty() 
? kConsumeTimeoutMs : 0));
+    std::unique_ptr<RdKafka::Message> msg(consume(messages.empty() ? 
std::min(kConsumeTimeoutMs, timeoutMs) : 0));
     switch (msg->err()) {
       case RdKafka::ERR_NO_ERROR: {
         const auto offset = msg->offset();
@@ -283,18 +311,13 @@ std::vector<std::unique_ptr<RdKafka::Message>> 
KafkaDataSource::poll(uint64_t ma
         break;
     }
 
-    if (!splitDone_ && std::chrono::steady_clock::now() >= deadline) {
-      if (!messages.empty()) {
-        return messages;
-      }
-      VELOX_FAIL(
-          "Timed out after {} ms fetching offset {} of {}-{}",
-          pollTimeoutMs,
-          nextOffset_,
-          split_->topic,
-          split_->partition);
+    if (!splitDone_ && pollTime() >= deadline) {
+      break;
     }
   }
+  if (!splitDone_ && messages.empty()) {
+    failOnTimeout();
+  }
   return messages;
 }
 
diff --git a/cpp/velox/operators/reader/KafkaDataSource.h 
b/cpp/velox/operators/reader/KafkaDataSource.h
index 59484b5e2a..22dde133ce 100644
--- a/cpp/velox/operators/reader/KafkaDataSource.h
+++ b/cpp/velox/operators/reader/KafkaDataSource.h
@@ -17,6 +17,7 @@
 
 #pragma once
 
+#include <chrono>
 #include <memory>
 #include <string>
 #include <vector>
@@ -64,6 +65,12 @@ class KafkaDataSource : public 
facebook::velox::connector::DataSource {
 
  private:
   friend class KafkaConnectorTest;
+  friend class KafkaPollTimeoutArgumentTest;
+
+  // Keep time and record consumption replaceable for deterministic timeout 
tests.
+  virtual std::chrono::steady_clock::time_point pollTime() const;
+
+  virtual RdKafka::Message* consume(int timeoutMs);
 
   // Polls up to 'maxRows' messages of the current split. Sets 'splitDone_' 
once the end offset is
   // reached.
diff --git a/cpp/velox/tests/CMakeLists.txt b/cpp/velox/tests/CMakeLists.txt
index a45420d395..d1685a8b34 100644
--- a/cpp/velox/tests/CMakeLists.txt
+++ b/cpp/velox/tests/CMakeLists.txt
@@ -182,5 +182,10 @@ if(ENABLE_ENHANCED_FEATURES)
 endif()
 
 if(ENABLE_KAFKA)
-  add_velox_test(velox_kafka_connector_test SOURCES KafkaConnectorTest.cc)
+  add_velox_test(
+    velox_kafka_connector_test
+    SOURCES
+    KafkaConnectorTest.cc
+    
${CMAKE_CURRENT_SOURCE_DIR}/../../../gluten-ut/native/KafkaPollTimeoutTest.cc
+  )
 endif()
diff --git a/gluten-ut/native/KafkaPollTimeoutTest.cc 
b/gluten-ut/native/KafkaPollTimeoutTest.cc
new file mode 100644
index 0000000000..45c3324319
--- /dev/null
+++ b/gluten-ut/native/KafkaPollTimeoutTest.cc
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <chrono>
+#include <cstdint>
+#include <deque>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <gtest/gtest.h>
+#include <librdkafka/rdkafka.h>
+#include <librdkafka/rdkafka_mock.h>
+#include <librdkafka/rdkafkacpp.h>
+
+#include "operators/reader/KafkaConnector.h"
+#include "operators/reader/KafkaDataSource.h"
+#include "velox/common/base/tests/GTestUtils.h"
+#include "velox/vector/tests/utils/VectorTestBase.h"
+
+using namespace facebook::velox;
+
+namespace gluten {
+namespace {
+
+constexpr char kConnectorId[] = "kafka-poll-timeout-test";
+constexpr char kTopic[] = "kafka-poll-timeout";
+constexpr int64_t kPollTimeoutMs = 10;
+
+} // namespace
+
+class KafkaPollTimeoutTest : public ::testing::Test,
+                             public test::VectorTestBase {
+protected:
+  static void SetUpTestCase() {
+    
memory::MemoryManager::testingSetInstance(memory::MemoryManager::Options{});
+  }
+
+  void SetUp() override {
+    auto *conf = rd_kafka_conf_new();
+    char error[512];
+    ASSERT_EQ(rd_kafka_conf_set(conf, "test.mock.num.brokers", "1", error,
+                                sizeof(error)),
+              RD_KAFKA_CONF_OK);
+    producer_ = rd_kafka_new(RD_KAFKA_PRODUCER, conf, error, sizeof(error));
+    ASSERT_NE(producer_, nullptr) << error;
+    cluster_ = rd_kafka_handle_mock_cluster(producer_);
+    ASSERT_NE(cluster_, nullptr);
+    ASSERT_EQ(rd_kafka_mock_topic_create(cluster_, kTopic, 1, 1),
+              RD_KAFKA_RESP_ERR_NO_ERROR);
+    // Keep broker responses outside both the configured timeout and the old 
100
+    // ms consume wait.
+    ASSERT_EQ(rd_kafka_mock_broker_set_rtt(cluster_, 1, 1'000),
+              RD_KAFKA_RESP_ERR_NO_ERROR);
+  }
+
+  void TearDown() override {
+    if (producer_ != nullptr) {
+      rd_kafka_destroy(producer_);
+    }
+  }
+
+  std::unique_ptr<KafkaDataSource> source() {
+    auto type = ROW({"offset"}, {BIGINT()});
+    connector::ColumnHandleMap columns{
+        {"offset", std::make_shared<KafkaColumnHandle>("offset", BIGINT())}};
+    return std::make_unique<KafkaDataSource>(type, columns, pool());
+  }
+
+  std::shared_ptr<KafkaConnectorSplit> split(int64_t endOffset) {
+    std::unordered_map<std::string, std::string> params{
+        {"bootstrap.servers", rd_kafka_mock_cluster_bootstraps(cluster_)}};
+    return std::make_shared<KafkaConnectorSplit>(kConnectorId, kTopic, 0, 0,
+                                                 endOffset, kPollTimeoutMs,
+                                                 true, std::move(params));
+  }
+
+  rd_kafka_t *producer_{nullptr};
+  rd_kafka_mock_cluster_t *cluster_{nullptr};
+};
+
+TEST_F(KafkaPollTimeoutTest, shortPollTimeoutReportsTimeout) {
+  auto reader = source();
+  reader->addSplit(split(1));
+  ContinueFuture future = ContinueFuture::makeEmpty();
+  VELOX_ASSERT_THROW(reader->next(1, future),
+                     "Timed out after 10 ms fetching offset");
+  EXPECT_EQ(reader->getCompletedRows(), 0);
+  EXPECT_EQ(reader->getCompletedBytes(), 0);
+}
+
+TEST_F(KafkaPollTimeoutTest, emptyRangeDoesNotWaitForBroker) {
+  auto reader = source();
+  reader->addSplit(split(0));
+  ContinueFuture future = ContinueFuture::makeEmpty();
+  EXPECT_EQ(reader->next(1, future).value(), nullptr);
+  EXPECT_EQ(reader->getCompletedRows(), 0);
+  EXPECT_EQ(reader->getCompletedBytes(), 0);
+}
+
+namespace {
+
+// Supply only the message fields used by poll(), without involving a broker or
+// vector allocation.
+class PollMessage final : public RdKafka::Message {
+public:
+  PollMessage(RdKafka::ErrorCode error, int64_t offset)
+      : error_(error), offset_(offset) {}
+
+  RdKafka::ErrorCode err() const override { return error_; }
+
+  std::string errstr() const override { return RdKafka::err2str(error_); }
+
+  int64_t offset() const override { return offset_; }
+
+  size_t len() const override { return 3; }
+
+  size_t key_len() const override { return 2; }
+
+  RdKafka::Topic *topic() const override { VELOX_UNREACHABLE(); }
+
+  std::string topic_name() const override { VELOX_UNREACHABLE(); }
+
+  int32_t partition() const override { VELOX_UNREACHABLE(); }
+
+  void *payload() const override { VELOX_UNREACHABLE(); }
+
+  const std::string *key() const override { VELOX_UNREACHABLE(); }
+
+  const void *key_pointer() const override { VELOX_UNREACHABLE(); }
+
+  RdKafka::MessageTimestamp timestamp() const override { VELOX_UNREACHABLE(); }
+
+  void *msg_opaque() const override { VELOX_UNREACHABLE(); }
+
+  int64_t latency() const override { VELOX_UNREACHABLE(); }
+
+  rd_kafka_message_t *c_ptr() override { VELOX_UNREACHABLE(); }
+
+  Status status() const override { VELOX_UNREACHABLE(); }
+
+  RdKafka::Headers *headers() override { VELOX_UNREACHABLE(); }
+
+  RdKafka::Headers *headers(RdKafka::ErrorCode * /*error*/) override {
+    VELOX_UNREACHABLE();
+  }
+
+  int32_t broker_id() const override { VELOX_UNREACHABLE(); }
+
+  int32_t leader_epoch() const override { VELOX_UNREACHABLE(); }
+
+  RdKafka::Error *offset_store() override { VELOX_UNREACHABLE(); }
+
+private:
+  const RdKafka::ErrorCode error_;
+  const int64_t offset_;
+};
+
+class ScriptedKafkaDataSource final : public KafkaDataSource {
+public:
+  using KafkaDataSource::KafkaDataSource;
+
+  struct Event {
+    RdKafka::ErrorCode error;
+    int64_t offset;
+    std::chrono::steady_clock::duration elapsed;
+  };
+
+  std::deque<Event> events;
+  std::vector<int> consumeTimeouts;
+
+private:
+  std::chrono::steady_clock::time_point pollTime() const override {
+    return now_;
+  }
+
+  RdKafka::Message *consume(int timeoutMs) override {
+    consumeTimeouts.push_back(timeoutMs);
+    VELOX_CHECK(!events.empty(), "Unexpected consume call");
+    const auto event = events.front();
+    events.pop_front();
+    now_ += event.elapsed;
+    return new PollMessage(event.error, event.offset);
+  }
+
+  std::chrono::steady_clock::time_point now_{};
+};
+
+} // namespace
+
+// Exercise the production poll loop with a controlled clock and recorded I/O
+// arguments. These tests neither sleep nor use wall-clock tolerances.
+class KafkaPollTimeoutArgumentTest : public ::testing::Test,
+                                     public test::VectorTestBase {
+protected:
+  static void SetUpTestCase() {
+    
memory::MemoryManager::testingSetInstance(memory::MemoryManager::Options{});
+  }
+
+  std::unique_ptr<ScriptedKafkaDataSource> source(int64_t timeoutMs,
+                                                  int64_t endOffset = 3) {
+    auto type = ROW({"offset"}, {BIGINT()});
+    connector::ColumnHandleMap columns{
+        {"offset", std::make_shared<KafkaColumnHandle>("offset", BIGINT())}};
+    auto reader =
+        std::make_unique<ScriptedKafkaDataSource>(type, columns, pool());
+    reader->split_ = std::make_shared<KafkaConnectorSplit>(
+        kConnectorId, kTopic, 0, 0, endOffset, timeoutMs, false,
+        std::unordered_map<std::string, std::string>{});
+    return reader;
+  }
+
+  std::vector<std::unique_ptr<RdKafka::Message>> poll(KafkaDataSource &reader,
+                                                      uint64_t maxRows = 1) {
+    return reader.poll(maxRows);
+  }
+
+  bool splitDone(const KafkaDataSource &reader) { return reader.splitDone_; }
+
+  int64_t nextOffset(const KafkaDataSource &reader) {
+    return reader.nextOffset_;
+  }
+};
+
+TEST_F(KafkaPollTimeoutArgumentTest, consumeUsesExactShortTimeout) {
+  auto reader = source(10);
+  reader->events = {
+      {RdKafka::ERR__TIMED_OUT, -1, std::chrono::milliseconds(10)}};
+  VELOX_ASSERT_THROW(poll(*reader), "Timed out after 10 ms fetching offset");
+  EXPECT_EQ(reader->consumeTimeouts, std::vector<int>({10}));
+}
+
+TEST_F(KafkaPollTimeoutArgumentTest, successiveConsumesShareDeadline) {
+  auto reader = source(250);
+  reader->events = {
+      {RdKafka::ERR__TIMED_OUT, -1, std::chrono::milliseconds(100)},
+      {RdKafka::ERR__TIMED_OUT, -1, std::chrono::milliseconds(100)},
+      {RdKafka::ERR__TIMED_OUT, -1, std::chrono::milliseconds(50)}};
+  VELOX_ASSERT_THROW(poll(*reader), "Timed out after 250 ms fetching offset");
+  EXPECT_EQ(reader->consumeTimeouts, std::vector<int>({100, 100, 50}));
+}
+
+TEST_F(KafkaPollTimeoutArgumentTest, collectedRecordsUseNonBlockingConsume) {
+  auto reader = source(300);
+  reader->events = {
+      {RdKafka::ERR_NO_ERROR, 0, std::chrono::milliseconds(25)},
+      {RdKafka::ERR__TIMED_OUT, -1, std::chrono::milliseconds(0)}};
+  const auto messages = poll(*reader, 2);
+  ASSERT_EQ(messages.size(), 1);
+  EXPECT_EQ(messages.front()->offset(), 0);
+  EXPECT_EQ(reader->consumeTimeouts, std::vector<int>({100, 0}));
+  EXPECT_EQ(reader->getCompletedBytes(), 5);
+}
+
+TEST_F(KafkaPollTimeoutArgumentTest, expiredBudgetReturnsCollectedRecord) {
+  auto reader = source(300);
+  reader->events = {{RdKafka::ERR_NO_ERROR, 0, 
std::chrono::milliseconds(301)}};
+  const auto messages = poll(*reader, 2);
+  ASSERT_EQ(messages.size(), 1);
+  EXPECT_EQ(messages.front()->offset(), 0);
+  EXPECT_EQ(reader->consumeTimeouts, std::vector<int>({100}));
+  EXPECT_EQ(reader->getCompletedBytes(), 5);
+  EXPECT_EQ(nextOffset(*reader), 1);
+  EXPECT_FALSE(splitDone(*reader));
+}
+
+} // namespace gluten


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to