This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new df4d33b747 [timeseries] Adding working E2E quickstart for
TimeSeriesEngineAuth (#16169)
df4d33b747 is described below
commit df4d33b74783058465c0a34cba483df790758ac8
Author: Shaurya Chaturvedi <[email protected]>
AuthorDate: Sun Jun 22 12:25:18 2025 -0700
[timeseries] Adding working E2E quickstart for TimeSeriesEngineAuth (#16169)
* [timeseries] Adding working E2E quickstart for TimeSeriesEngineAuth
* Added license header
* Added script info to time series quickstart classes
* Addressed comments
---------
Co-authored-by: Shaurya Chaturvedi <[email protected]>
---
.../org/apache/pinot/tools/AuthQuickstart.java | 39 +--------
.../tools/TimeSeriesEngineAuthQuickStart.java | 62 ++++++++++++++
.../pinot/tools/TimeSeriesEngineQuickStart.java | 10 ++-
.../tools/admin/command/QuickstartRunner.java | 7 ++
.../{AuthQuickstart.java => utils/AuthUtils.java} | 45 +++--------
.../main/resources/scripts/timeseries/query.txt | 4 +
.../resources/scripts/timeseries/run_ts_query.py | 94 ++++++++++++++++++++++
7 files changed, 190 insertions(+), 71 deletions(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
index 4930c8f6b0..303d10bce7 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
@@ -26,6 +26,7 @@ import org.apache.pinot.common.auth.AuthProviderUtils;
import org.apache.pinot.common.auth.BasicAuthUtils;
import org.apache.pinot.spi.auth.AuthProvider;
import org.apache.pinot.spi.plugin.PluginManager;
+import org.apache.pinot.tools.utils.AuthUtils;
public class AuthQuickstart extends Quickstart {
@@ -42,43 +43,7 @@ public class AuthQuickstart extends Quickstart {
@Override
public Map<String, Object> getConfigOverrides() {
Map<String, Object> properties = new HashMap<>(super.getConfigOverrides());
-
- // controller
- properties.put("pinot.controller.segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
- properties.put("controller.admin.access.control.factory.class",
-
"org.apache.pinot.controller.api.access.BasicAuthAccessControlFactory");
- properties.put("controller.admin.access.control.principals", "admin, user,
service, tableonly");
-
properties.put("controller.admin.access.control.principals.admin.password",
"verysecret");
-
properties.put("controller.admin.access.control.principals.service.password",
"verysecrettoo");
- properties.put("controller.admin.access.control.principals.user.password",
"secret");
-
properties.put("controller.admin.access.control.principals.user.permissions",
"READ");
-
properties.put("controller.admin.access.control.principals.tableonly.password",
"secrettoo");
-
properties.put("controller.admin.access.control.principals.tableonly.permissions",
"READ");
-
properties.put("controller.admin.access.control.principals.tableonly.tables",
"baseballStats");
-
- // broker
- properties.put("pinot.broker.access.control.class",
"org.apache.pinot.broker.broker.BasicAuthAccessControlFactory");
- properties.put("pinot.broker.access.control.principals", "admin, user,
service, tableonly");
- properties.put("pinot.broker.access.control.principals.admin.password",
"verysecret");
- properties.put("pinot.broker.access.control.principals.service.password",
"verysecrettoo");
- properties.put("pinot.broker.access.control.principals.user.password",
"secret");
-
properties.put("pinot.broker.access.control.principals.tableonly.password",
"secrettoo");
- properties.put("pinot.broker.access.control.principals.tableonly.tables",
"baseballStats");
-
- // server
- properties.put("pinot.server.segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
- properties.put("pinot.server.segment.uploader.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
- properties.put("pinot.server.instance.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
-
- // minion
- properties.put("segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
- properties.put("task.auth.token", "Basic YWRtaW46dmVyeXNlY3JldA==");
-
- // loggers
- properties.put("pinot.controller.logger.root.dir", "logs");
- properties.put("pinot.broker.logger.root.dir", "logs");
- properties.put("pinot.server.logger.root.dir", "logs");
- properties.put("pinot.minion.logger.root.dir", "logs");
+ properties.putAll(AuthUtils.getAuthQuickStartDefaultConfigs());
return properties;
}
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/TimeSeriesEngineAuthQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/TimeSeriesEngineAuthQuickStart.java
new file mode 100644
index 0000000000..f34e9c2e08
--- /dev/null
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/TimeSeriesEngineAuthQuickStart.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.tools;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.pinot.common.auth.AuthProviderUtils;
+import org.apache.pinot.common.auth.BasicAuthUtils;
+import org.apache.pinot.spi.auth.AuthProvider;
+import org.apache.pinot.spi.plugin.PluginManager;
+import org.apache.pinot.tools.utils.AuthUtils;
+
+
+/**
+ * Quick start for Time Series Engine with authentication.
+ *
+ * To test this quick start, you can run the following command in
pinot-tools/src/main/resources/scripts/timeseries/
+ * python3 run_ts_query.py --auth
+ */
+public class TimeSeriesEngineAuthQuickStart extends TimeSeriesEngineQuickStart
{
+
+ @Override
+ public List<String> types() {
+ return Collections.singletonList("TIME_SERIES_AUTH");
+ }
+
+ @Override
+ public AuthProvider getAuthProvider() {
+ return
AuthProviderUtils.makeAuthProvider(BasicAuthUtils.toBasicAuthToken("admin",
"verysecret"));
+ }
+
+ @Override
+ public Map<String, Object> getConfigOverrides() {
+ Map<String, Object> properties = new HashMap<>(super.getConfigOverrides());
+ properties.putAll(AuthUtils.getAuthQuickStartDefaultConfigs());
+ return properties;
+ }
+
+ public static void main(String[] args) throws Exception {
+ PluginManager.get().init();
+ new TimeSeriesEngineAuthQuickStart().execute();
+ printStatus(Color.GREEN, "Default login credential to login controller is
admin/verysecret.");
+ }
+}
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/TimeSeriesEngineQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/TimeSeriesEngineQuickStart.java
index b64bec82f8..2a81a5cce2 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/TimeSeriesEngineQuickStart.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/TimeSeriesEngineQuickStart.java
@@ -29,7 +29,12 @@ import org.apache.pinot.tools.admin.command.QuickstartRunner;
import org.apache.pinot.tsdb.spi.PinotTimeSeriesConfiguration;
import org.apache.pinot.tsdb.spi.series.SimpleTimeSeriesBuilderFactory;
-
+/**
+ * Quick start for Time Series Engine.
+ *
+ * To test this quick start, you can run the following command in
pinot-tools/src/main/resources/scripts/timeseries/
+ * python3 run_ts_query.py
+ */
public class TimeSeriesEngineQuickStart extends Quickstart {
private static final String[] TIME_SERIES_TABLE_DIRECTORIES = new String[]{
"examples/batch/airlineStats",
@@ -77,7 +82,8 @@ public class TimeSeriesEngineQuickStart extends Quickstart {
Preconditions.checkState(quickstartRunnerDir.mkdirs());
List<QuickstartTableRequest> quickstartTableRequests =
bootstrapStreamTableDirectories(quickstartTmpDir);
final QuickstartRunner runner =
- new QuickstartRunner(quickstartTableRequests, 1, 1, 2, 1,
quickstartRunnerDir, getConfigOverrides());
+ new QuickstartRunner(quickstartTableRequests, 1, 1, 2, 1,
quickstartRunnerDir, getConfigOverrides(),
+ getAuthProvider());
startKafka();
startAllDataStreams(_kafkaStarter, quickstartTmpDir);
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
index 85a878b936..2d396ec46a 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
@@ -91,6 +91,13 @@ public class QuickstartRunner {
true, Map.of());
}
+ public QuickstartRunner(List<QuickstartTableRequest> tableRequests, int
numControllers, int numBrokers,
+ int numServers, int numMinions, File tempDir, Map<String, Object>
configOverrides, AuthProvider authProvider)
+ throws Exception {
+ this(tableRequests, numControllers, numBrokers, numServers, numMinions,
tempDir, true, authProvider,
+ configOverrides, null, true, Map.of());
+ }
+
public QuickstartRunner(List<QuickstartTableRequest> tableRequests, int
numControllers, int numBrokers,
int numServers, int numMinions, File tempDir, boolean enableIsolation,
AuthProvider authProvider,
Map<String, Object> configOverrides, String zkExternalAddress, boolean
deleteExistingData,
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/utils/AuthUtils.java
similarity index 66%
copy from pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
copy to pinot-tools/src/main/java/org/apache/pinot/tools/utils/AuthUtils.java
index 4930c8f6b0..82081b990a 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/AuthQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/utils/AuthUtils.java
@@ -16,37 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.tools;
+package org.apache.pinot.tools.utils;
-import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
-import org.apache.pinot.common.auth.AuthProviderUtils;
-import org.apache.pinot.common.auth.BasicAuthUtils;
-import org.apache.pinot.spi.auth.AuthProvider;
-import org.apache.pinot.spi.plugin.PluginManager;
-public class AuthQuickstart extends Quickstart {
- @Override
- public List<String> types() {
- return Collections.singletonList("AUTH");
- }
+public class AuthUtils {
+ private static final String DEFAULT_AUTH_TOKEN = "Basic
YWRtaW46dmVyeXNlY3JldA==";
- @Override
- public AuthProvider getAuthProvider() {
- return
AuthProviderUtils.makeAuthProvider(BasicAuthUtils.toBasicAuthToken("admin",
"verysecret"));
+ private AuthUtils() {
}
- @Override
- public Map<String, Object> getConfigOverrides() {
- Map<String, Object> properties = new HashMap<>(super.getConfigOverrides());
-
+ public static Map<String, Object> getAuthQuickStartDefaultConfigs() {
+ Map<String, Object> properties = new HashMap<>();
// controller
- properties.put("pinot.controller.segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
+ properties.put("pinot.controller.segment.fetcher.auth.token",
DEFAULT_AUTH_TOKEN);
properties.put("controller.admin.access.control.factory.class",
-
"org.apache.pinot.controller.api.access.BasicAuthAccessControlFactory");
+ "org.apache.pinot.controller.api.access.BasicAuthAccessControlFactory");
properties.put("controller.admin.access.control.principals", "admin, user,
service, tableonly");
properties.put("controller.admin.access.control.principals.admin.password",
"verysecret");
properties.put("controller.admin.access.control.principals.service.password",
"verysecrettoo");
@@ -66,13 +53,13 @@ public class AuthQuickstart extends Quickstart {
properties.put("pinot.broker.access.control.principals.tableonly.tables",
"baseballStats");
// server
- properties.put("pinot.server.segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
- properties.put("pinot.server.segment.uploader.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
- properties.put("pinot.server.instance.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
+ properties.put("pinot.server.segment.fetcher.auth.token",
DEFAULT_AUTH_TOKEN);
+ properties.put("pinot.server.segment.uploader.auth.token",
DEFAULT_AUTH_TOKEN);
+ properties.put("pinot.server.instance.auth.token", DEFAULT_AUTH_TOKEN);
// minion
- properties.put("segment.fetcher.auth.token", "Basic
YWRtaW46dmVyeXNlY3JldA==");
- properties.put("task.auth.token", "Basic YWRtaW46dmVyeXNlY3JldA==");
+ properties.put("segment.fetcher.auth.token", DEFAULT_AUTH_TOKEN);
+ properties.put("task.auth.token", DEFAULT_AUTH_TOKEN);
// loggers
properties.put("pinot.controller.logger.root.dir", "logs");
@@ -81,10 +68,4 @@ public class AuthQuickstart extends Quickstart {
properties.put("pinot.minion.logger.root.dir", "logs");
return properties;
}
-
- public static void main(String[] args)
- throws Exception {
- PluginManager.get().init();
- new AuthQuickstart().execute();
- }
}
diff --git a/pinot-tools/src/main/resources/scripts/timeseries/query.txt
b/pinot-tools/src/main/resources/scripts/timeseries/query.txt
new file mode 100644
index 0000000000..25b5e4564f
--- /dev/null
+++ b/pinot-tools/src/main/resources/scripts/timeseries/query.txt
@@ -0,0 +1,4 @@
+fetch{table="meetupRsvp_REALTIME",filter="",ts_column="__metadata$recordTimestamp",ts_unit="MILLISECONDS",value="1"}
+ | max{group_city}
+ | transformNull{0}
+ | keepLastValue{}
\ No newline at end of file
diff --git a/pinot-tools/src/main/resources/scripts/timeseries/run_ts_query.py
b/pinot-tools/src/main/resources/scripts/timeseries/run_ts_query.py
new file mode 100644
index 0000000000..c261992e76
--- /dev/null
+++ b/pinot-tools/src/main/resources/scripts/timeseries/run_ts_query.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import argparse
+import requests
+import time
+import logging
+import sys
+
+# === Constants ===
+DEFAULT_QUERY_FILE = "query.txt"
+DEFAULT_API_URL = "http://localhost:8000/timeseries/api/v1/query_range";
+DEFAULT_DURATION_SECONDS = 3600
+AUTH_TOKEN = "Basic YWRtaW46dmVyeXNlY3JldA=="
+
+# === Logger Setup ===
+def setup_logger():
+ logger = logging.getLogger("TimeseriesQuery")
+ handler = logging.StreamHandler(sys.stdout)
+ formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+ handler.setFormatter(formatter)
+ logger.setLevel(logging.INFO)
+ logger.addHandler(handler)
+ return logger
+
+# === File Reader ===
+def read_query_file(path, logger):
+ try:
+ with open(path, 'r') as f:
+ return f.read()
+ except Exception as e:
+ logger.error(f"Failed to read query file '{path}': {e}")
+ sys.exit(1)
+
+# === Request Builder ===
+def build_request(query, start, end, token=None):
+ params = {
+ 'language': 'm3ql',
+ 'start': start,
+ 'end': end,
+ 'query': query
+ }
+ headers = {}
+ if token:
+ headers['Authorization'] = token
+ return params, headers
+
+# === Main Entry ===
+def main():
+ parser = argparse.ArgumentParser(description="Query a time series API
endpoint.")
+ parser.add_argument('--query-file', default=DEFAULT_QUERY_FILE,
help=f'Path to the query file (default: {DEFAULT_QUERY_FILE})')
+ parser.add_argument('--url', default=DEFAULT_API_URL, help=f'API endpoint
URL (default: {DEFAULT_API_URL})')
+ parser.add_argument('--auth', action='store_true', help='Enable
Authorization header using default token')
+ parser.add_argument('--duration', type=int,
default=DEFAULT_DURATION_SECONDS, help=f'Query time range duration in seconds
(default: {DEFAULT_DURATION_SECONDS})')
+ args = parser.parse_args()
+
+ logger = setup_logger()
+ logger.info(f"Using query file: {args.query_file}")
+ query = read_query_file(args.query_file, logger)
+
+ start_time = int(time.time())
+ end_time = start_time + args.duration
+
+ token = AUTH_TOKEN if args.auth else None
+ params, headers = build_request(query, start_time, end_time, token)
+
+ try:
+ logger.info(f"Sending request to {args.url}")
+ response = requests.get(args.url, params=params, headers=headers)
+ logger.info(f"Response text: {response.text}")
+ response.raise_for_status()
+ except requests.RequestException as e:
+ logger.error(f"Request failed: {e}")
+ sys.exit(1)
+
+if __name__ == '__main__':
+ main()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
