This is an automated email from the ASF dual-hosted git repository.
swaminathanmanish pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2ea51e9d63e Fix table-level segment metadata silently omitting
segments when replication < servers (#20182)
2ea51e9d63e is described below
commit 2ea51e9d63e269f5c8969f47de5e7cd28fa33583
Author: Manak Bisht <[email protected]>
AuthorDate: Tue Nov 10 21:15:57 2026 +0530
Fix table-level segment metadata silently omitting segments when
replication < servers (#20182)
* Fix table-level segment metadata silently omitting segments when
replication < servers
The table-level path of TableMetadataReader#getSegmentsMetadata passed the
server -> endpoint map from getDataInstanceAdminEndpoints straight to
CompletionServiceHelper, which expects endpoint -> server. Each response's
server lookup returned null, and with multiRequestPerServer=false every body
was stored under the same null key, overwriting the previous one. The call
reported no failures, so there was no fallback, and the API returned 200
with
only the last server's segments (roughly replication/N of the table).
Pass endpoints.inverse() as the legacy per-segment path already does, and
rename the misleading endpointToInstance variable in
getDataInstanceAdminEndpoints to instanceToEndpoint.
Add TableMetadataReaderTest covering full, partial and no replication across
multiple fake servers.
* Fix 'Server: null' in error logs of PinotTableReloadStatusReporter and
PinotTableRestletResource by passing the inverted endpoint map
---
.../api/resources/PinotTableRestletResource.java | 4 +-
.../helix/core/PinotHelixResourceManager.java | 6 +-
.../services/PinotTableReloadStatusReporter.java | 2 +-
.../pinot/controller/util/TableMetadataReader.java | 2 +-
.../controller/util/TableMetadataReaderTest.java | 176 +++++++++++++++++++++
5 files changed, 183 insertions(+), 7 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
index 1046b2a17c8..34b24cdff06 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableRestletResource.java
@@ -1428,7 +1428,7 @@ public class PinotTableRestletResource {
BiMap<String, String> serverEndPoints =
_pinotHelixResourceManager.getDataInstanceAdminEndpoints(serverToSegments.keySet());
CompletionServiceHelper completionServiceHelper =
- new CompletionServiceHelper(_executor, _connectionManager,
serverEndPoints,
+ new CompletionServiceHelper(_executor, _connectionManager,
serverEndPoints.inverse(),
_pinotHelixResourceManager.getServerAdminAuthProvider());
List<String> serverUrls = new ArrayList<>();
@@ -1597,7 +1597,7 @@ public class PinotTableRestletResource {
BiMap<String, String> serverEndPoints =
_pinotHelixResourceManager.getDataInstanceAdminEndpoints(serverToSegments.keySet());
CompletionServiceHelper completionServiceHelper =
- new CompletionServiceHelper(_executor, _connectionManager,
serverEndPoints,
+ new CompletionServiceHelper(_executor, _connectionManager,
serverEndPoints.inverse(),
_pinotHelixResourceManager.getServerAdminAuthProvider());
List<String> serverUrls = new ArrayList<>();
BiMap<String, String> endpointsToServers = serverEndPoints.inverse();
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index 6c200112508..874b0879809 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -4519,7 +4519,7 @@ public class PinotHelixResourceManager {
/// server instances. With BiMap, both mappings are easily available
public BiMap<String, String> getDataInstanceAdminEndpoints(Set<String>
instances)
throws InvalidConfigException {
- BiMap<String, String> endpointToInstance =
HashBiMap.create(instances.size());
+ BiMap<String, String> instanceToEndpoint =
HashBiMap.create(instances.size());
for (String instance : instances) {
String instanceAdminEndpoint;
try {
@@ -4531,9 +4531,9 @@ public class PinotHelixResourceManager {
LOGGER.error(errorMessage, e);
throw new InvalidConfigException(errorMessage);
}
- endpointToInstance.put(instance, instanceAdminEndpoint);
+ instanceToEndpoint.put(instance, instanceAdminEndpoint);
}
- return endpointToInstance;
+ return instanceToEndpoint;
}
/// Returns the service identity provider used for outbound Server admin API
requests.
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/services/PinotTableReloadStatusReporter.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/services/PinotTableReloadStatusReporter.java
index 8e0019bfe91..8fecfb5efa2 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/services/PinotTableReloadStatusReporter.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/services/PinotTableReloadStatusReporter.java
@@ -166,7 +166,7 @@ public class PinotTableReloadStatusReporter {
final List<String> serverUrls = getServerUrls(serverEndPoints,
reloadJobMetadata, serverToSegments);
final CompletionServiceHelper completionServiceHelper =
- new CompletionServiceHelper(_executor, _connectionManager,
serverEndPoints,
+ new CompletionServiceHelper(_executor, _connectionManager,
serverEndPoints.inverse(),
_pinotHelixResourceManager.getServerAdminAuthProvider());
final CompletionServiceHelper.CompletionServiceResponse serviceResponse =
completionServiceHelper.doMultiGetRequest(serverUrls, null, true,
10000);
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
index 7b6389862a4..71ba2bf8c40 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
@@ -203,7 +203,7 @@ public class TableMetadataReader {
try {
List<String> tableUrls = buildTableLevelUrls(serverToSegs, endpoints,
tableNameWithType, columns, segments, reader);
- return fetchAndAggregateMetadata(tableUrls, endpoints,
/*perSegmentJson=*/false,
+ return fetchAndAggregateMetadata(tableUrls, endpoints.inverse(),
/*perSegmentJson=*/false,
tableNameWithType, timeoutMs);
} catch (RuntimeException e) {
log.warn("Failed to fetch table metadata for table {} using new server
endpoint, falling back to legacy "
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/util/TableMetadataReaderTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/util/TableMetadataReaderTest.java
new file mode 100644
index 00000000000..e2fd78e9fde
--- /dev/null
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/util/TableMetadataReaderTest.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.controller.util;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
+import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.mockito.ArgumentMatchers.anySet;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+
+
+public class TableMetadataReaderTest {
+ private static final String TABLE_NAME = "testTable_OFFLINE";
+ private static final int TIMEOUT_MS = 10_000;
+
+ private final ExecutorService _executor = Executors.newFixedThreadPool(4);
+ private final PoolingHttpClientConnectionManager _connectionManager = new
PoolingHttpClientConnectionManager();
+ private final List<HttpServer> _httpServers = new ArrayList<>();
+ private final AtomicInteger _legacyRequestCount = new AtomicInteger();
+
+ @BeforeClass
+ public void setUp() {
+ _connectionManager.setDefaultMaxPerRoute(4);
+ }
+
+ @AfterMethod
+ public void stopServers() {
+ _httpServers.forEach(server -> server.stop(0));
+ _httpServers.clear();
+ _legacyRequestCount.set(0);
+ }
+
+ @AfterClass
+ public void tearDown() {
+ _executor.shutdownNow();
+ _connectionManager.close();
+ }
+
+ /// Every server holds every segment, so any single server's response is
already complete.
+ @Test
+ public void testTableLevelMetadataWithFullReplication()
+ throws Exception {
+ Map<String, List<String>> serverToSegments = new LinkedHashMap<>();
+ serverToSegments.put("server0", List.of("seg0", "seg1", "seg2"));
+ serverToSegments.put("server1", List.of("seg0", "seg1", "seg2"));
+ serverToSegments.put("server2", List.of("seg0", "seg1", "seg2"));
+
+ JsonNode metadata = getSegmentsMetadata(serverToSegments);
+
+ assertEquals(segmentNames(metadata), Set.of("seg0", "seg1", "seg2"));
+ assertEquals(_legacyRequestCount.get(), 0);
+ }
+
+ /// 4 servers with replication 2: each server holds only half of the
segments, so every server's response is needed.
+ @Test
+ public void testTableLevelMetadataWithPartialReplication()
+ throws Exception {
+ Map<String, List<String>> serverToSegments = new LinkedHashMap<>();
+ serverToSegments.put("server0", List.of("seg0", "seg1"));
+ serverToSegments.put("server1", List.of("seg1", "seg2"));
+ serverToSegments.put("server2", List.of("seg2", "seg3"));
+ serverToSegments.put("server3", List.of("seg3", "seg0"));
+
+ JsonNode metadata = getSegmentsMetadata(serverToSegments);
+
+ assertEquals(segmentNames(metadata), Set.of("seg0", "seg1", "seg2",
"seg3"));
+ assertEquals(_legacyRequestCount.get(), 0);
+ }
+
+ /// Each server holds a distinct segment, so dropping any server's response
loses its segment.
+ @Test
+ public void testTableLevelMetadataWithoutReplication()
+ throws Exception {
+ Map<String, List<String>> serverToSegments = new LinkedHashMap<>();
+ serverToSegments.put("server0", List.of("seg0"));
+ serverToSegments.put("server1", List.of("seg1"));
+ serverToSegments.put("server2", List.of("seg2"));
+
+ JsonNode metadata = getSegmentsMetadata(serverToSegments);
+
+ assertEquals(segmentNames(metadata), Set.of("seg0", "seg1", "seg2"));
+ assertEquals(_legacyRequestCount.get(), 0);
+ }
+
+ private JsonNode getSegmentsMetadata(Map<String, List<String>>
serverToSegments)
+ throws Exception {
+ BiMap<String, String> serverToEndpoint = HashBiMap.create();
+ for (Map.Entry<String, List<String>> entry : serverToSegments.entrySet()) {
+ HttpServer httpServer = startServer(entry.getValue());
+ serverToEndpoint.put(entry.getKey(), "http://localhost:" +
httpServer.getAddress().getPort());
+ }
+
+ PinotHelixResourceManager helixResourceManager =
mock(PinotHelixResourceManager.class);
+
when(helixResourceManager.getServerToSegmentsMap(TABLE_NAME)).thenReturn(serverToSegments);
+
when(helixResourceManager.getDataInstanceAdminEndpoints(anySet())).thenReturn(serverToEndpoint);
+
+ TableMetadataReader reader = new TableMetadataReader(_executor,
_connectionManager, helixResourceManager);
+ return reader.getSegmentsMetadata(TABLE_NAME, null, null, TIMEOUT_MS);
+ }
+
+ /// Starts a fake server whose table-level endpoint returns metadata for the
given segments. The legacy per-segment
+ /// endpoint returns 404, so tests can assert that the table-level path was
used.
+ private HttpServer startServer(List<String> segments)
+ throws IOException {
+ ObjectNode tableLevelResponse = JsonUtils.newObjectNode();
+ for (String segment : segments) {
+ tableLevelResponse.set(segment,
JsonUtils.newObjectNode().put("segmentName", segment));
+ }
+ byte[] tableLevelBody =
JsonUtils.objectToString(tableLevelResponse).getBytes(StandardCharsets.UTF_8);
+
+ HttpServer httpServer = HttpServer.create(new
InetSocketAddress("localhost", 0), 0);
+ httpServer.createContext("/tables/", exchange -> {
+ String path = exchange.getRequestURI().getPath();
+ if (path.equals("/tables/" + TABLE_NAME + "/segments/metadata")) {
+ exchange.sendResponseHeaders(200, tableLevelBody.length);
+ try (OutputStream out = exchange.getResponseBody()) {
+ out.write(tableLevelBody);
+ }
+ } else {
+ _legacyRequestCount.incrementAndGet();
+ exchange.sendResponseHeaders(404, -1);
+ exchange.close();
+ }
+ });
+ httpServer.start();
+ _httpServers.add(httpServer);
+ return httpServer;
+ }
+
+ private static Set<String> segmentNames(JsonNode metadata) {
+ Set<String> segmentNames = new HashSet<>();
+ metadata.fieldNames().forEachRemaining(segmentNames::add);
+ return segmentNames;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]