This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5f86b5e19a add /tasks/{taskType}/{tableNameWithType}/debug API (#8949)
5f86b5e19a is described below
commit 5f86b5e19a195449806689cef3332c94f27dfd10
Author: Haitao Zhang <hai...@startree.ai>
AuthorDate: Wed Jun 22 14:07:47 2022 -0700
add /tasks/{taskType}/{tableNameWithType}/debug API (#8949)
* add /tasks/{taskType}/{tableNameWithType}/debug API
* address comments
* add the missing word
* revise java docs
---
.../api/resources/PinotTaskRestletResource.java | 22 ++++++++++-
.../core/minion/PinotHelixTaskResourceManager.java | 46 +++++++++++++++++++++-
2 files changed, 65 insertions(+), 3 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java
index d7d8a161d5..42614777a6 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java
@@ -210,17 +210,35 @@ public class PinotTaskRestletResource {
@ApiOperation("Fetch information for all the tasks for the given task type")
public Map<String, PinotHelixTaskResourceManager.TaskDebugInfo>
getTasksDebugInfo(
@ApiParam(value = "Task type", required = true) @PathParam("taskType")
String taskType,
- @ApiParam(value = "verbosity (By default, prints for running and error
tasks. Value of >0 prints for all tasks)")
+ @ApiParam(value = "verbosity (Prints information for all the tasks for
the given task type."
+ + "By default, only prints subtask details for running and error
tasks. "
+ + "Value of > 0 prints subtask details for all tasks)")
@DefaultValue("0") @QueryParam("verbosity") int verbosity) {
return _pinotHelixTaskResourceManager.getTasksDebugInfo(taskType,
verbosity);
}
+ @GET
+ @Path("/tasks/{taskType}/{tableNameWithType}/debug")
+ @ApiOperation("Fetch information for all the tasks for the given task type
and table")
+ public Map<String, PinotHelixTaskResourceManager.TaskDebugInfo>
getTasksDebugInfo(
+ @ApiParam(value = "Task type", required = true) @PathParam("taskType")
String taskType,
+ @ApiParam(value = "Table name with type", required = true)
@PathParam("tableNameWithType")
+ String tableNameWithType,
+ @ApiParam(value = "verbosity (Prints information for all the tasks for
the given task type and table."
+ + "By default, only prints subtask details for running and error
tasks. "
+ + "Value of > 0 prints subtask details for all tasks)")
+ @DefaultValue("0") @QueryParam("verbosity") int verbosity) {
+ return _pinotHelixTaskResourceManager.getTasksDebugInfoByTable(taskType,
tableNameWithType, verbosity);
+ }
+
@GET
@Path("/tasks/task/{taskName}/debug")
@ApiOperation("Fetch information for the given task name")
public PinotHelixTaskResourceManager.TaskDebugInfo getTaskDebugInfo(
@ApiParam(value = "Task name", required = true) @PathParam("taskName")
String taskName,
- @ApiParam(value = "verbosity (By default, prints for running and error
tasks. Value of >0 prints for all tasks)")
+ @ApiParam(value = "verbosity (Prints information for the given task
name."
+ + "By default, only prints subtask details for running and error
tasks. "
+ + "Value of > 0 prints subtask details for all tasks)")
@DefaultValue("0") @QueryParam("verbosity") int verbosity) {
return _pinotHelixTaskResourceManager.getTaskDebugInfo(taskName,
verbosity);
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java
index 56e69220a4..233c513b33 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java
@@ -460,7 +460,7 @@ public class PinotHelixTaskResourceManager {
* Helper method to return a map of task names to corresponding task state
* where the task corresponds to the given Pinot table name. This is used to
* check status of all tasks for a given table.
- * @param taskType Task Name
+ * @param taskType Pinot taskType / Helix JobQueue
* @param tableNameWithType table name with type to filter on
* @return Map of filtered task name to corresponding state
*/
@@ -531,6 +531,50 @@ public class PinotHelixTaskResourceManager {
return taskDebugInfos;
}
+ /**
+ * Given a taskType and a tableNameWithType, helper method to debug all the
HelixJobs for
+ * the taskType and tableNameWithType. For each of the HelixJobs, collects
status of
+ * the (sub)tasks in the taskbatch.
+ *
+ * @param taskType Pinot taskType / Helix JobQueue
+ * @param tableNameWithType Table name with type to filter on
+ * @param verbosity By default, does not show details for completed tasks.
+ * If verbosity > 0, shows details for all tasks.
+ * @return Map of Pinot Task Name to TaskDebugInfo. TaskDebugInfo contains
details for subtasks.
+ */
+ public synchronized Map<String, TaskDebugInfo> getTasksDebugInfoByTable(
+ String taskType, String tableNameWithType, int verbosity) {
+ Map<String, TaskDebugInfo> taskDebugInfos = new TreeMap<>();
+ WorkflowContext workflowContext =
_taskDriver.getWorkflowContext(getHelixJobQueueName(taskType));
+ if (workflowContext == null) {
+ return taskDebugInfos;
+ }
+
+ Map<String, TaskState> helixJobStates = workflowContext.getJobStates();
+ for (String helixJobName : helixJobStates.keySet()) {
+ String pinotTaskName = getPinotTaskName(helixJobName);
+
+ // Iterate through all task configs associated with this task name
+ for (PinotTaskConfig taskConfig : getTaskConfigs(pinotTaskName)) {
+ Map<String, String> pinotConfigs = taskConfig.getConfigs();
+
+ // Filter task configs that matches this table name
+ if (pinotConfigs != null) {
+ String tableNameConfig = pinotConfigs.get(TABLE_NAME);
+ if (tableNameConfig != null &&
tableNameConfig.equals(tableNameWithType)) {
+ // Found a match. Add task debug info to the result
+ // TODO: If a task consists of subtasks belonging to the current
table and other tables at the same time,
+ // we will collect debug info of them all. We may want to filter
out debug info that does not belong
+ // to the given table.
+ taskDebugInfos.put(pinotTaskName,
getTaskDebugInfo(workflowContext, helixJobName, verbosity));
+ break;
+ }
+ }
+ }
+ }
+ return taskDebugInfos;
+ }
+
/**
* Given a taskName, collects status of the (sub)tasks in the taskName.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
