This is an automated email from the ASF dual-hosted git repository.
yashmayya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1d8b147f6e Allow overriding cluster configs in quickstarts (#14919)
1d8b147f6e is described below
commit 1d8b147f6e56664536af4ef2910f8ad8813d913f
Author: Yash Mayya <[email protected]>
AuthorDate: Sat Feb 8 21:08:45 2025 +0700
Allow overriding cluster configs in quickstarts (#14919)
---
.../pinot/tools/MultistageEngineQuickStart.java | 7 +++++++
.../org/apache/pinot/tools/QuickStartBase.java | 4 ++++
.../java/org/apache/pinot/tools/Quickstart.java | 2 +-
.../tools/admin/command/QuickstartRunner.java | 23 ++++++++++++++++++++--
4 files changed, 33 insertions(+), 3 deletions(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
index e249c37c0b..447544aa87 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/MultistageEngineQuickStart.java
@@ -297,6 +297,13 @@ public class MultistageEngineQuickStart extends Quickstart
{
return configOverrides;
}
+ @Override
+ protected Map<String, String> getClusterConfigOverrides() {
+ return Map.of(
+
CommonConstants.Helix.CONFIG_OF_MULTI_STAGE_ENGINE_MAX_SERVER_QUERY_THREADS,
"50"
+ );
+ }
+
@Override
protected int getNumQuickstartRunnerServers() {
return 3;
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java
index b8296e4093..388192d623 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java
@@ -291,6 +291,10 @@ public abstract class QuickStartBase {
}
}
+ protected Map<String, String> getClusterConfigOverrides() {
+ return Map.of();
+ }
+
protected String[] getDefaultBatchTableDirectories() {
return DEFAULT_OFFLINE_TABLE_DIRECTORIES;
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
index b5894ca968..0ebb0053b7 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java
@@ -67,7 +67,7 @@ public class Quickstart extends QuickStartBase {
QuickstartRunner runner =
new QuickstartRunner(quickstartTableRequests, 1, 1,
getNumQuickstartRunnerServers(), 1, quickstartRunnerDir,
- true, getAuthProvider(), getConfigOverrides(), _zkExternalAddress,
true);
+ true, getAuthProvider(), getConfigOverrides(), _zkExternalAddress,
true, getClusterConfigOverrides());
printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker,
server and minion *****");
runner.startAll();
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
index 77cf329e0d..85a878b936 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
@@ -29,6 +29,11 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.model.HelixConfigScope;
+import org.apache.helix.model.builder.HelixConfigScopeBuilder;
import org.apache.pinot.spi.auth.AuthProvider;
import org.apache.pinot.spi.config.tenant.TenantRole;
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -69,6 +74,7 @@ public class QuickstartRunner {
private final boolean _enableTenantIsolation;
private final AuthProvider _authProvider;
private final Map<String, Object> _configOverrides;
+ private final Map<String, String> _clusterConfigOverrides;
private final boolean _deleteExistingData;
// If this field is non-null, an embedded Zookeeper instance will not be
launched
@@ -82,12 +88,13 @@ public class QuickstartRunner {
int numServers, int numMinions, File tempDir, Map<String, Object>
configOverrides)
throws Exception {
this(tableRequests, numControllers, numBrokers, numServers, numMinions,
tempDir, true, null, configOverrides, null,
- true);
+ true, Map.of());
}
public QuickstartRunner(List<QuickstartTableRequest> tableRequests, int
numControllers, int numBrokers,
int numServers, int numMinions, File tempDir, boolean enableIsolation,
AuthProvider authProvider,
- Map<String, Object> configOverrides, String zkExternalAddress, boolean
deleteExistingData)
+ Map<String, Object> configOverrides, String zkExternalAddress, boolean
deleteExistingData,
+ Map<String, String> clusterConfigOverrides)
throws Exception {
_tableRequests = tableRequests;
_numControllers = numControllers;
@@ -98,6 +105,7 @@ public class QuickstartRunner {
_enableTenantIsolation = enableIsolation;
_authProvider = authProvider;
_configOverrides = new HashMap<>(configOverrides);
+ _clusterConfigOverrides = clusterConfigOverrides;
if (numMinions > 0) {
// configure the controller to schedule tasks when minion is enabled
_configOverrides.put("controller.task.scheduler.enabled", true);
@@ -127,11 +135,22 @@ public class QuickstartRunner {
.setTenantIsolation(_enableTenantIsolation)
.setDataDir(new File(_tempDir, DEFAULT_CONTROLLER_DIR +
i).getAbsolutePath())
.setConfigOverrides(_configOverrides);
+
if (!controllerStarter.execute()) {
throw new RuntimeException("Failed to start Controller");
}
_controllerPorts.add(DEFAULT_CONTROLLER_PORT + i);
}
+
+ HelixManager helixManager =
+ HelixManagerFactory.getZKHelixManager(CLUSTER_NAME, "localhost_" +
_controllerPorts.get(0),
+ InstanceType.CONTROLLER, _zkExternalAddress != null ?
_zkExternalAddress : ZK_ADDRESS);
+ helixManager.connect();
+ HelixConfigScope scope =
+ new
HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER).forCluster(CLUSTER_NAME).build();
+ for (Map.Entry<String, String> entry : _clusterConfigOverrides.entrySet())
{
+ helixManager.getConfigAccessor().set(scope, entry.getKey(),
entry.getValue());
+ }
}
private void startBrokers()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
