This is an automated email from the ASF dual-hosted git repository.
shauryachats pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 94711630d28 Create /tableConfigs/tune endpoint that validates and
tunes TableConfigs and returns the tuned config. (#18056)
94711630d28 is described below
commit 94711630d28f1b00578a53dbfd647720da3eac97
Author: Rekha Seethamraju <[email protected]>
AuthorDate: Tue May 19 14:25:41 2026 -0700
Create /tableConfigs/tune endpoint that validates and tunes TableConfigs
and returns the tuned config. (#18056)
---
.../api/resources/TableConfigsRestletResource.java | 56 +++++++++++++---
.../api/TableConfigsRestletResourceTest.java | 78 ++++++++++++++++++++++
.../utils/builder/ControllerRequestURLBuilder.java | 4 ++
3 files changed, 129 insertions(+), 9 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
index 87f27e0d38c..e37ee4d086f 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
@@ -240,13 +240,13 @@ public class TableConfigsRestletResource {
}
if (offlineTableConfig != null) {
- tuneConfig(offlineTableConfig, schema);
+ applyTuning(offlineTableConfig, schema);
if (!ignoreActiveTasks) {
PinotTableRestletResource.tableTasksValidation(offlineTableConfig,
_pinotHelixTaskResourceManager);
}
}
if (realtimeTableConfig != null) {
- tuneConfig(realtimeTableConfig, schema);
+ applyTuning(realtimeTableConfig, schema);
if (!ignoreActiveTasks) {
PinotTableRestletResource.tableTasksValidation(realtimeTableConfig,
_pinotHelixTaskResourceManager);
}
@@ -408,7 +408,7 @@ public class TableConfigsRestletResource {
LOGGER.info("Updated schema: {}", tableName);
if (offlineTableConfig != null) {
- tuneConfig(offlineTableConfig, schema);
+ applyTuning(offlineTableConfig, schema);
if (_pinotHelixResourceManager.hasOfflineTable(tableName)) {
_pinotHelixResourceManager.updateTableConfig(offlineTableConfig,
forceTableSchemaUpdate);
LOGGER.info("Updated offline table config: {}", tableName);
@@ -418,7 +418,7 @@ public class TableConfigsRestletResource {
}
}
if (realtimeTableConfig != null) {
- tuneConfig(realtimeTableConfig, schema);
+ applyTuning(realtimeTableConfig, schema);
if (_pinotHelixResourceManager.hasRealtimeTable(tableName)) {
_pinotHelixResourceManager.updateTableConfig(realtimeTableConfig,
forceTableSchemaUpdate);
LOGGER.info("Updated realtime table config: {}", tableName);
@@ -460,6 +460,47 @@ public class TableConfigsRestletResource {
+ "(ALL|TASK|UPSERT|TENANT|MINION_INSTANCES|ACTIVE_TASKS)")
@QueryParam("validationTypesToSkip") @Nullable String typesToSkip,
@Context HttpHeaders httpHeaders,
@Context Request request) {
+ Pair<TableConfigs, Map<String, Object>> tableConfigsAndUnrecognizedProps =
+ parseAndValidateTableConfigs(tableConfigsStr, typesToSkip,
httpHeaders, request);
+ TableConfigs tableConfigs = tableConfigsAndUnrecognizedProps.getLeft();
+ ObjectNode response = JsonUtils.objectToJsonNode(tableConfigs).deepCopy();
+ response.set("unrecognizedProperties",
JsonUtils.objectToJsonNode(tableConfigsAndUnrecognizedProps.getRight()));
+ return response.toString();
+ }
+
+ /**
+ * Validates and tunes the {@link TableConfigs} as provided in the
tableConfigsStr json, by applying tuner configs,
+ * ensuring min replicas and storage quota constraints, and returns the
tuned TableConfigs.
+ */
+ @POST
+ @Path("/tableConfigs/tune")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Tune the TableConfigs",
+ notes = "Validates and applies tuning (tuner configs, min replicas,
storage quota) to the TableConfigs, "
+ + "returning the result that would be stored on create/update")
+ @ManualAuthorization // performed after parsing TableConfigs
+ public String tuneConfig(String tableConfigsStr,
+ @ApiParam(value = "comma separated list of validation type(s) to skip.
supported types: "
+ + "(ALL|TASK|UPSERT|TENANT|MINION_INSTANCES|ACTIVE_TASKS)")
+ @QueryParam("validationTypesToSkip") @Nullable String typesToSkip,
@Context HttpHeaders httpHeaders,
+ @Context Request request) {
+ Pair<TableConfigs, Map<String, Object>> tableConfigsAndUnrecognizedProps =
+ parseAndValidateTableConfigs(tableConfigsStr, typesToSkip,
httpHeaders, request);
+ TableConfigs tableConfigs = tableConfigsAndUnrecognizedProps.getLeft();
+ Schema schema = tableConfigs.getSchema();
+ if (tableConfigs.getOffline() != null) {
+ applyTuning(tableConfigs.getOffline(), schema);
+ }
+ if (tableConfigs.getRealtime() != null) {
+ applyTuning(tableConfigs.getRealtime(), schema);
+ }
+ ObjectNode response = JsonUtils.objectToJsonNode(tableConfigs).deepCopy();
+ response.set("unrecognizedProperties",
JsonUtils.objectToJsonNode(tableConfigsAndUnrecognizedProps.getRight()));
+ return response.toString();
+ }
+
+ private Pair<TableConfigs, Map<String, Object>>
parseAndValidateTableConfigs(String tableConfigsStr,
+ @Nullable String typesToSkip, HttpHeaders httpHeaders, Request request) {
Pair<TableConfigs, Map<String, Object>> tableConfigsAndUnrecognizedProps;
try {
tableConfigsAndUnrecognizedProps =
@@ -482,13 +523,10 @@ public class TableConfigsRestletResource {
if (!accessControl.hasAccess(httpHeaders, TargetType.TABLE, rawTableName,
Actions.Table.VALIDATE_TABLE_CONFIGS)) {
throw new ControllerApplicationException(LOGGER, "Permission denied",
Response.Status.FORBIDDEN);
}
-
- ObjectNode response = JsonUtils.objectToJsonNode(tableConfigs).deepCopy();
- response.set("unrecognizedProperties",
JsonUtils.objectToJsonNode(tableConfigsAndUnrecognizedProps.getRight()));
- return response.toString();
+ return tableConfigsAndUnrecognizedProps;
}
- private void tuneConfig(TableConfig tableConfig, Schema schema) {
+ private void applyTuning(TableConfig tableConfig, Schema schema) {
TableConfigTunerUtils.applyTunerConfigs(_pinotHelixResourceManager,
tableConfig, schema, Collections.emptyMap());
TableConfigUtils.ensureMinReplicas(tableConfig,
_controllerConf.getDefaultTableMinReplicas());
TableConfigUtils.ensureStorageQuotaConstraints(tableConfig,
_controllerConf.getDimTableMaxSize());
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
index 75552355c60..1e3e4e4b4c1 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
@@ -698,6 +698,84 @@ public class TableConfigsRestletResourceTest extends
ControllerTest {
adminClient.getSchemaClient().deleteSchema(tableName);
}
+ @Test
+ public void testTuneConfig()
+ throws IOException {
+ String tuneConfigUrl =
DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsTune();
+
+ String tableName = "testTune";
+ TableConfigs tableConfigs;
+
+ // invalid json
+ try {
+ tableConfigs = new TableConfigs(tableName, createDummySchema(tableName),
createOfflineTableConfig(tableName),
+ createRealtimeTableConfig(tableName));
+ sendPostRequest(tuneConfigUrl,
tableConfigs.toPrettyJsonString().replace("\"offline\"", "offline\""));
+ fail("Tune of a TableConfigs with invalid json string should have
failed");
+ } catch (Exception e) {
+ // expected
+ }
+
+ // null table configs
+ try {
+ tableConfigs = new TableConfigs(tableName, createDummySchema(tableName),
null, null);
+ sendPostRequest(tuneConfigUrl, tableConfigs.toPrettyJsonString());
+ fail("Tune of a TableConfigs with null offline and realtime tableConfig
should have failed");
+ } catch (Exception e) {
+ // expected
+ }
+
+ // replicas are bumped up to min replicas
+ String tableName1 = "testTuneReplicas";
+ TableConfig replicaTestOfflineTableConfig =
createOfflineTableConfig(tableName1);
+ TableConfig replicaTestRealtimeTableConfig =
createRealtimeTableConfig(tableName1);
+ replicaTestOfflineTableConfig.getValidationConfig().setReplication("1");
+ replicaTestRealtimeTableConfig.getValidationConfig().setReplication("1");
+ tableConfigs = new TableConfigs(tableName1, createDummySchema(tableName1),
replicaTestOfflineTableConfig,
+ replicaTestRealtimeTableConfig);
+ String response = sendPostRequest(tuneConfigUrl,
tableConfigs.toPrettyJsonString());
+ TableConfigs tuned = JsonUtils.stringToObject(response,
TableConfigs.class);
+ Assert.assertEquals(tuned.getOffline().getReplication(),
DEFAULT_MIN_NUM_REPLICAS);
+ Assert.assertEquals(tuned.getRealtime().getReplication(),
DEFAULT_MIN_NUM_REPLICAS);
+
+ // dim table storage quota is capped
+ String tableName2 = "testTuneQuota";
+ TableConfig offlineDimTableConfig =
createOfflineDimTableConfig(tableName2);
+ tableConfigs = new TableConfigs(tableName2,
createDummySchemaWithPrimaryKey(tableName2), offlineDimTableConfig,
+ null);
+ response = sendPostRequest(tuneConfigUrl,
tableConfigs.toPrettyJsonString());
+ tuned = JsonUtils.stringToObject(response, TableConfigs.class);
+ Assert.assertEquals(tuned.getOffline().getQuotaConfig().getStorage(),
+ DEFAULT_INSTANCE.getControllerConfig().getDimTableMaxSize());
+
+ // tuner configs are applied
+ String tableName3 = "testTuneTunerConfig";
+ Schema schema3 = createDummySchema(tableName3);
+ tableConfigs = new TableConfigs(tableName3, schema3,
createOfflineTunerTableConfig(tableName3),
+ createRealtimeTunerTableConfig(tableName3));
+ response = sendPostRequest(tuneConfigUrl,
tableConfigs.toPrettyJsonString());
+ tuned = JsonUtils.stringToObject(response, TableConfigs.class);
+
Assert.assertTrue(tuned.getOffline().getIndexingConfig().getInvertedIndexColumns()
+ .containsAll(schema3.getDimensionNames()));
+
Assert.assertTrue(tuned.getOffline().getIndexingConfig().getNoDictionaryColumns()
+ .containsAll(schema3.getMetricNames()));
+
Assert.assertTrue(tuned.getRealtime().getIndexingConfig().getInvertedIndexColumns()
+ .containsAll(schema3.getDimensionNames()));
+
Assert.assertTrue(tuned.getRealtime().getIndexingConfig().getNoDictionaryColumns()
+ .containsAll(schema3.getMetricNames()));
+
+ // response includes unrecognizedProperties
+ String tableName4 = "testTuneUnrecognized";
+ TableConfig offlineTableConfig = createOfflineTableConfig(tableName4);
+ tableConfigs = new TableConfigs(tableName4, createDummySchema(tableName4),
offlineTableConfig, null);
+ ObjectNode tableConfigsJson =
JsonUtils.objectToJsonNode(tableConfigs).deepCopy();
+ tableConfigsJson.put("illegalKey1", 1);
+ response = sendPostRequest(tuneConfigUrl,
tableConfigsJson.toPrettyString());
+ JsonNode responseJson = JsonUtils.stringToJsonNode(response);
+ Assert.assertTrue(responseJson.has("unrecognizedProperties"));
+
Assert.assertTrue(responseJson.get("unrecognizedProperties").has("/illegalKey1"));
+ }
+
@Test
public void testValidateConfigWithClusterValidationSkipTypes()
throws Exception {
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
index a7fa0901982..4346cc394b9 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
@@ -409,6 +409,10 @@ public class ControllerRequestURLBuilder {
return StringUtil.join("/", _baseUrl, "tableConfigs", "validate");
}
+ public String forTableConfigsTune() {
+ return StringUtil.join("/", _baseUrl, "tableConfigs", "tune");
+ }
+
public String forSegmentReload(String tableName, String segmentName, boolean
forceDownload) {
return StringUtil.join("/", _baseUrl, "segments", tableName,
encode(segmentName),
"reload?forceDownload=" + forceDownload);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
