This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 22a59516168 Document job management and interactive API protos (#39294)
22a59516168 is described below
commit 22a595161680ed170ebc196deb30f8f7adbbd2e3
Author: Danny McCormick <[email protected]>
AuthorDate: Wed Jul 22 07:57:50 2026 -0400
Document job management and interactive API protos (#39294)
Add documentation comments to previously undocumented messages and fields
in the Beam job management and interactive API protos:
beam_artifact_api.proto: ArtifactRetrievalService, ArtifactStagingService,
ArtifactRequestWrapper, ProxyManifest, PutArtifactMetadata, and more.
beam_expansion_api.proto: ExpansionRequest/Response,
DiscoverSchemaTransformRequest/Response, SchemaTransformConfig.
beam_job_api.proto: JobService, JobInfo, JobStateEvent, JobMessage,
GetJobMetricsRequest/Response, MetricResults, PipelineOptionType.
beam_interactive_api.proto: TestStreamService, EventsRequest.
---
.../interactive/v1/beam_interactive_api.proto | 4 +++
.../job_management/v1/beam_artifact_api.proto | 19 +++++++++++--
.../job_management/v1/beam_expansion_api.proto | 9 ++++++
.../model/job_management/v1/beam_job_api.proto | 32 ++++++++++++++++++++++
4 files changed, 62 insertions(+), 2 deletions(-)
diff --git
a/model/interactive/src/main/proto/org/apache/beam/model/interactive/v1/beam_interactive_api.proto
b/model/interactive/src/main/proto/org/apache/beam/model/interactive/v1/beam_interactive_api.proto
index 6351b247257..faae8026140 100644
---
a/model/interactive/src/main/proto/org/apache/beam/model/interactive/v1/beam_interactive_api.proto
+++
b/model/interactive/src/main/proto/org/apache/beam/model/interactive/v1/beam_interactive_api.proto
@@ -50,11 +50,15 @@ message TestStreamFileRecord {
org.apache.beam.model.pipeline.v1.TestStreamPayload.Event recorded_event = 1;
}
+// A gRPC service that serves cached TestStream events to a TestStream source.
+// Used by Interactive Beam to replay recorded pipeline elements from a
streaming cache.
service TestStreamService {
// A TestStream will request for events using this RPC.
rpc Events(EventsRequest) returns (stream
org.apache.beam.model.pipeline.v1.TestStreamPayload.Event) {}
}
+// A request for TestStream events from a streaming cache.
+// The client specifies which PCollection output tags to read events for.
message EventsRequest {
// The set of PCollections to read from. These are the PTransform outputs
// local names. These are a subset of the TestStream's outputs. This allows
diff --git
a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_artifact_api.proto
b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_artifact_api.proto
index 35cd4f0c73d..a64352c9289 100644
---
a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_artifact_api.proto
+++
b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_artifact_api.proto
@@ -78,18 +78,23 @@ message ResolveArtifactsResponse {
// A request to get an artifact.
message GetArtifactRequest {
+ // (Required) The artifact to retrieve, typically a resolved artifact from
ResolveArtifacts.
org.apache.beam.model.pipeline.v1.ArtifactInformation artifact = 1;
}
// Part of a response to getting an artifact.
message GetArtifactResponse {
+ // A chunk of the artifact's binary content.
bytes data = 1;
}
// Wraps an ArtifactRetrievalService request for use in
ReverseArtifactRetrievalService.
+// The server sends these requests to the client during the reverse staging
flow.
message ArtifactRequestWrapper {
oneof request {
+ // Request to resolve a set of artifacts into concrete artifact references.
ResolveArtifactsRequest resolve_artifact = 1000;
+ // Request to retrieve the bytes of a specific artifact.
GetArtifactRequest get_artifact = 1001;
}
}
@@ -151,16 +156,22 @@ message ArtifactMetadata {
// A collection of artifacts.
message Manifest {
+ // (Required) The list of artifacts staged for a job.
repeated ArtifactMetadata artifact = 1;
}
// A manifest with location information.
message ProxyManifest {
+ // (Required) The manifest of staged artifacts.
Manifest manifest = 1;
+ // A location (name + URI) at which a staged artifact can be retrieved.
message Location {
- string name = 1;
- string uri = 2;
+ // (Required) The name of the artifact, matching the name in the Manifest.
+ string name = 1;
+ // (Required) The URI at which the artifact content can be retrieved.
+ string uri = 2;
}
+ // (Required) The list of artifact locations for retrieval.
repeated Location location = 2;
}
@@ -173,6 +184,7 @@ message GetManifestRequest {
// A response containing a job manifest.
message GetManifestResponse {
+ // (Required) The manifest of staged artifacts for the job.
Manifest manifest = 1;
}
@@ -187,9 +199,11 @@ message LegacyGetArtifactRequest {
// Part of an artifact.
message ArtifactChunk {
+ // A chunk of the artifact's binary content.
bytes data = 1;
}
+// Metadata for an artifact being staged, sent as the first message in a
PutArtifact stream.
message PutArtifactMetadata {
// (Required) A token for artifact staging session. This token can be
obtained
// from PrepareJob request in JobService
@@ -211,6 +225,7 @@ message PutArtifactRequest {
}
}
+// A response to staging an artifact. An empty response indicating the
artifact was staged successfully.
message PutArtifactResponse {
}
diff --git
a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto
b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto
index a4736f8b493..0e3b1adbffd 100644
---
a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto
+++
b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto
@@ -33,6 +33,8 @@ import "google/protobuf/struct.proto";
import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto";
import "org/apache/beam/model/pipeline/v1/schema.proto";
+// A request to expand a single PTransform in a remote SDK. The expansion
service
+// resolves the transform into its constituent subtransforms and outputs.
message ExpansionRequest {
// Set of components needed to interpret the transform, or which
// may be useful for its expansion. This includes the input
@@ -64,6 +66,8 @@ message ExpansionRequest {
google.protobuf.Struct pipeline_options = 6;
}
+// The result of expanding a PTransform, containing the expanded transform and
+// all newly created components.
message ExpansionResponse {
// Set of components needed to execute the expanded transform,
// including the (original) inputs, outputs, and subtransforms.
@@ -82,9 +86,13 @@ message ExpansionResponse {
string error = 10;
}
+// A request to discover all SchemaTransformProviders registered with the
expansion service.
+// This is used by cross-language pipeline construction to enumerate available
transforms.
message DiscoverSchemaTransformRequest {
}
+// Configuration metadata for a SchemaTransform, describing its inputs,
outputs,
+// and configuration schema. Used to enable cross-language transform discovery.
message SchemaTransformConfig {
// Config schema of the SchemaTransform
org.apache.beam.model.pipeline.v1.Schema config_schema = 1;
@@ -102,6 +110,7 @@ message SchemaTransformConfig {
string description = 4;
}
+// A response containing all discovered SchemaTransform configurations.
message DiscoverSchemaTransformResponse {
// A mapping from SchemaTransform ID to schema transform config of discovered
// SchemaTransforms
diff --git
a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_job_api.proto
b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_job_api.proto
index 8e196d5bedd..353eaec5799 100644
---
a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_job_api.proto
+++
b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_job_api.proto
@@ -112,6 +112,7 @@ message RunJobRequest {
}
+// The response from submitting a job for execution.
message RunJobResponse {
string job_id = 1; // (required) The ID for the executing job
}
@@ -144,6 +145,7 @@ message DrainJobResponse {
}
// A subset of info provided by ProvisionApi.ProvisionInfo
+// Represents metadata about a submitted job.
message JobInfo {
string job_id = 1; // (required)
string job_name = 2; // (required)
@@ -155,6 +157,7 @@ message JobInfo {
// Throws error GRPC_STATUS_UNAVAILABLE if server is down
message GetJobsRequest { }
+// The response from requesting a list of all invoked jobs.
message GetJobsResponse {
repeated JobInfo job_info = 1; // (required)
}
@@ -168,6 +171,7 @@ message GetJobStateRequest {
}
+// An event representing a job state transition. Emitted by GetState and
GetStateStream.
message JobStateEvent {
JobState.Enum state = 1; // (required)
google.protobuf.Timestamp timestamp = 2; // (required)
@@ -182,6 +186,7 @@ message GetJobPipelineRequest {
}
+// The response from requesting a job's pipeline representation.
message GetJobPipelineResponse {
org.apache.beam.model.pipeline.v1.Pipeline pipeline = 1; // (required)
}
@@ -195,25 +200,41 @@ message JobMessagesRequest {
string job_id = 1; // (required)
}
+// A single log message or diagnostic event from a running job.
message JobMessage {
+ // (Required) A unique identifier for this message.
string message_id = 1;
+ // (Required) The time at which the message was emitted, as a string
representation.
string time = 2;
+ // (Required) The importance level of this message.
MessageImportance importance = 3;
+ // (Required) The text content of the message.
string message_text = 4;
+ // Importance levels for job messages, ordered from least to most severe.
enum MessageImportance {
+ // The importance was not specified.
MESSAGE_IMPORTANCE_UNSPECIFIED = 0;
+ // Debug-level messages, typically very verbose.
JOB_MESSAGE_DEBUG = 1;
+ // Detailed informational messages.
JOB_MESSAGE_DETAILED = 2;
+ // Basic informational messages.
JOB_MESSAGE_BASIC = 3;
+ // Warning messages indicating potential issues.
JOB_MESSAGE_WARNING = 4;
+ // Error messages indicating failures.
JOB_MESSAGE_ERROR = 5;
}
}
+// A streaming response from GetMessageStream, containing either a job message
+// or a job state change event.
message JobMessagesResponse {
oneof response {
+ // A log message or diagnostic event from the job.
JobMessage message_response = 1;
+ // A job state transition event.
JobStateEvent state_response = 2;
}
}
@@ -270,17 +291,22 @@ message JobState {
}
+// A request to fetch metrics for a given job.
message GetJobMetricsRequest {
string job_id = 1; // (required)
}
+// A response containing metrics for a given job.
message GetJobMetricsResponse {
+ // (Required) The metrics results containing both attempted and committed
values.
MetricResults metrics = 1;
}
// All metrics for a given job. Runners may support one or the other or both.
message MetricResults {
+ // Metrics reflecting the result of attempted (non-committed) computations.
repeated org.apache.beam.model.pipeline.v1.MonitoringInfo attempted = 1;
+ // Metrics reflecting the result of committed computations.
repeated org.apache.beam.model.pipeline.v1.MonitoringInfo committed = 2;
}
@@ -296,12 +322,17 @@ message DescribePipelineOptionsRequest {
// Types mirror those of JSON, since that's how pipeline options are
serialized.
message PipelineOptionType {
enum Enum {
+ // A string value.
STRING = 0;
+ // A boolean (true/false) value.
BOOLEAN = 1;
// whole numbers, see
https://json-schema.org/understanding-json-schema/reference/numeric.html
INTEGER = 2;
+ // A floating-point number.
NUMBER = 3;
+ // An array of values.
ARRAY = 4;
+ // A nested object.
OBJECT = 5;
};
}
@@ -324,6 +355,7 @@ message PipelineOptionDescriptor {
string group = 5;
}
+// The response from describing pipeline options, containing a list of option
descriptors.
message DescribePipelineOptionsResponse {
// List of pipeline option descriptors.
repeated PipelineOptionDescriptor options = 1;
