This is an automated email from the ASF dual-hosted git repository. nehapawar pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push: new 11272b7eb2 Added optional force param to the table configs update API (#10441) 11272b7eb2 is described below commit 11272b7eb25b915df617b783f28b06fb5ef9c87d Author: Ragesh Rajagopalan <ragesh.rajagopa...@gmail.com> AuthorDate: Thu Mar 23 17:25:58 2023 -0700 Added optional force param to the table configs update API (#10441) --- .../api/resources/PinotSchemaRestletResource.java | 2 +- .../api/resources/TableConfigsRestletResource.java | 11 ++++-- .../helix/core/PinotHelixResourceManager.java | 8 ++--- .../api/TableConfigsRestletResourceTest.java | 40 ++++++++++++++++++++++ .../pinot/controller/helix/TableCacheTest.java | 2 +- 5 files changed, 54 insertions(+), 9 deletions(-) diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java index 4104256140..66d26c1264 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSchemaRestletResource.java @@ -396,7 +396,7 @@ public class PinotSchemaRestletResource { } try { - _pinotHelixResourceManager.updateSchema(schema, reload); + _pinotHelixResourceManager.updateSchema(schema, reload, false); // Best effort notification. If controller fails at this point, no notification is given. LOGGER.info("Notifying metadata event for updating schema: {}", schemaName); _metadataEventNotifierFactory.create().notifyOnSchemaEvents(schema, SchemaEventType.UPDATE); diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java index 8d8daff4d5..336aa3472e 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java @@ -291,6 +291,9 @@ public class TableConfigsRestletResource { * Updated the {@link TableConfigs} by updating the schema tableName, * then updating the offline tableConfig or creating a new one if it doesn't already exist in the cluster, * then updating the realtime tableConfig or creating a new one if it doesn't already exist in the cluster. + * + * The option to skip table config validation (validationTypesToSkip) and force update the table schema + * (forceTableSchemaUpdate) are provided for testing purposes and should be used with caution. */ @PUT @Path("/tableConfigs/{tableName}") @@ -304,8 +307,10 @@ public class TableConfigsRestletResource { @ApiParam(value = "comma separated list of validation type(s) to skip. supported types: (ALL|TASK|UPSERT)") @QueryParam("validationTypesToSkip") @Nullable String typesToSkip, @ApiParam(value = "Reload the table if the new schema is backward compatible") @DefaultValue("false") - @QueryParam("reload") boolean reload, String tableConfigsStr) - throws Exception { + @QueryParam("reload") boolean reload, + @ApiParam(value = "Force update the table schema") @DefaultValue("false") + @QueryParam("forceTableSchemaUpdate") boolean forceTableSchemaUpdate, + String tableConfigsStr) throws Exception { Pair<TableConfigs, Map<String, Object>> tableConfigsAndUnrecognizedProps; TableConfigs tableConfigs; try { @@ -333,7 +338,7 @@ public class TableConfigsRestletResource { Schema schema = tableConfigs.getSchema(); try { - _pinotHelixResourceManager.updateSchema(schema, reload); + _pinotHelixResourceManager.updateSchema(schema, reload, forceTableSchemaUpdate); LOGGER.info("Updated schema: {}", tableName); if (offlineTableConfig != null) { diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java index 0cb468a04d..84ef273224 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java @@ -1332,7 +1332,7 @@ public class PinotHelixResourceManager { } } - public void updateSchema(Schema schema, boolean reload) + public void updateSchema(Schema schema, boolean reload, boolean forceTableSchemaUpdate) throws SchemaNotFoundException, SchemaBackwardIncompatibleException, TableNotFoundException { String schemaName = schema.getSchemaName(); LOGGER.info("Updating schema: {} with reload: {}", schemaName, reload); @@ -1342,7 +1342,7 @@ public class PinotHelixResourceManager { throw new SchemaNotFoundException(String.format("Schema: %s does not exist", schemaName)); } - updateSchema(schema, oldSchema, false); + updateSchema(schema, oldSchema, forceTableSchemaUpdate); if (reload) { LOGGER.info("Reloading tables with name: {}", schemaName); @@ -1357,7 +1357,7 @@ public class PinotHelixResourceManager { * Helper method to update the schema, or throw SchemaBackwardIncompatibleException when the new schema is not * backward-compatible with the existing schema. */ - private void updateSchema(Schema schema, Schema oldSchema, boolean force) + private void updateSchema(Schema schema, Schema oldSchema, boolean forceTableSchemaUpdate) throws SchemaBackwardIncompatibleException { String schemaName = schema.getSchemaName(); schema.updateBooleanFieldsIfNeeded(oldSchema); @@ -1367,7 +1367,7 @@ public class PinotHelixResourceManager { } boolean isBackwardCompatible = schema.isBackwardCompatibleWith(oldSchema); if (!isBackwardCompatible) { - if (force) { + if (forceTableSchemaUpdate) { LOGGER.warn("Force updated schema: {} which is backward incompatible with the existing schema", oldSchema); } else { // TODO: Add the reason of the incompatibility diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java index a506f38ac3..2ee5bb1361 100644 --- a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java +++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java @@ -485,6 +485,46 @@ public class TableConfigsRestletResourceTest extends ControllerTest { sendDeleteRequest(DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsDelete(tableName)); } + @Test + public void testForceUpdateTableSchemaAndConfigs() + throws IOException { + String tableName = "testUpdate1"; + TableConfig offlineTableConfig = createOfflineTableConfig(tableName); + Schema schema = createDummySchema(tableName); + TableConfigs tableConfigs = new TableConfigs(tableName, schema, offlineTableConfig, null); + + sendPostRequest(_createTableConfigsUrl, tableConfigs.toPrettyJsonString()); + String response = sendGetRequest(DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsGet(tableName)); + TableConfigs tableConfigsResponse = JsonUtils.stringToObject(response, TableConfigs.class); + Assert.assertNotNull(tableConfigs.getOffline()); + + // Remove field from schema and try to update schema without the 'forceTableSchemaUpdate' option + schema.removeField("dimA"); + tableConfigs = + new TableConfigs(tableName, schema, tableConfigsResponse.getOffline(), tableConfigsResponse.getRealtime()); + + String tableConfigUpdateUrl = DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsUpdate(tableName); + try { + sendPutRequest(tableConfigUpdateUrl, tableConfigs.toPrettyJsonString()); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains("is not backward-compatible with the existing schema")); + } + + // Skip validate table configs – Exception is still thrown + String newTableConfigUpdateUrl = tableConfigUpdateUrl + "?validationTypesToSkip=ALL"; + try { + sendPutRequest(newTableConfigUpdateUrl, tableConfigs.toPrettyJsonString()); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().contains("is not backward-compatible with the existing schema")); + } + + // Skip table config validation as well as force update the table schema – no exceptions are thrown + newTableConfigUpdateUrl = tableConfigUpdateUrl + "?validationTypesToSkip=ALL&forceTableSchemaUpdate=true"; + response = sendPutRequest(newTableConfigUpdateUrl, tableConfigs.toPrettyJsonString()); + Assert.assertTrue(response.contains("TableConfigs updated for testUpdate1")); + sendDeleteRequest(DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsDelete(tableName)); + } + @Test public void testDeleteConfig() throws Exception { diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/TableCacheTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/TableCacheTest.java index 9f2024dd99..53697a7627 100644 --- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/TableCacheTest.java +++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/TableCacheTest.java @@ -128,7 +128,7 @@ public class TableCacheTest { // Update the schema schema.addField(new DimensionFieldSpec("newColumn", DataType.LONG, true)); - TEST_INSTANCE.getHelixResourceManager().updateSchema(schema, false); + TEST_INSTANCE.getHelixResourceManager().updateSchema(schema, false, false); // Wait for at most 10 seconds for the callback to update the schema in the cache // NOTE: // - Schema should never be null during the transitioning --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org