Repository: beam Updated Branches: refs/heads/master 763fb50b5 -> 9151676a5
Add ability to bundle any message with components in Runner API Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/368ad236 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/368ad236 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/368ad236 Branch: refs/heads/master Commit: 368ad236ca03bd011d2f207bcc34f5961290c0a0 Parents: cd4e6e4 Author: Kenneth Knowles <k...@google.com> Authored: Thu Feb 16 14:07:40 2017 -0800 Committer: Kenneth Knowles <k...@google.com> Committed: Thu Feb 16 15:18:24 2017 -0800 ---------------------------------------------------------------------- .../src/main/proto/beam_runner_api.proto | 66 ++++++++++++++++---- 1 file changed, 54 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/368ad236/sdks/common/runner-api/src/main/proto/beam_runner_api.proto ---------------------------------------------------------------------- diff --git a/sdks/common/runner-api/src/main/proto/beam_runner_api.proto b/sdks/common/runner-api/src/main/proto/beam_runner_api.proto index a9133ab..f5dc81d 100644 --- a/sdks/common/runner-api/src/main/proto/beam_runner_api.proto +++ b/sdks/common/runner-api/src/main/proto/beam_runner_api.proto @@ -30,17 +30,9 @@ option java_outer_classname = "RunnerApi"; import "google/protobuf/any.proto"; -// A Pipeline is a hierarchical graph of PTransforms, linked -// by PCollections. -// -// This is represented by a number of by-reference maps to nodes, -// PCollections, SDK environments, UDF, etc., for -// supporting compact reuse and arbitrary graph structure. -// -// All of the keys in the maps here are arbitrary strings that are only -// required to be internally consistent within this proto message. -message Pipeline { - +// A set of mappings from id to message. This is included as an optional field +// on any proto message that may contain references needing resolution. +message Components { // (Required) A map from pipeline-scoped id to GraphNode<PTransform, PCollection> // // Each node is required to contain a PTransform specification. @@ -61,9 +53,58 @@ message Pipeline { // (Required) A map from pipeline-scoped id to FunctionSpec, // a record for a particular user-defined function. map<string, FunctionSpec> function_specs = 6; +} + +// A disjoint union of all the things that may contain references +// that require Components to resolve. +message MessageWithComponents { + + // (Optional) The by-reference components of the root message, + // enabling a standalone message. + // + // If this is absent, it is expected that there are no + // references. + Components components = 1; + + // (Required) The root message that may contain pointers + // that should be resolved by looking inside components. + oneof root { + Coder coder = 2; + CombinePayload combine_payload = 3; + FunctionSpec function_spec = 4; + GraphNode graph_node = 5; + ParDoPayload par_do_payload = 6; + PTransform ptransform = 7; + PCollection pcollection = 8; + ReadPayload read_payload = 9; + SdkFunctionSpec sdk_function_spec = 10; + SideInput side_input = 11; + WindowIntoPayload window_into_payload = 12; + WindowingStrategy windowing_strategy = 13; + UrnWithParameter urn_with_parameter = 14; + } +} + +// A Pipeline is a hierarchical graph of PTransforms, linked +// by PCollections. +// +// This is represented by a number of by-reference maps to nodes, +// PCollections, SDK environments, UDF, etc., for +// supporting compact reuse and arbitrary graph structure. +// +// All of the keys in the maps here are arbitrary strings that are only +// required to be internally consistent within this proto message. +message Pipeline { + + // (Required) The coders, UDFs, graph nodes, etc, that make up + // this pipeline. + Components components = 1; + + // (Required) The graph node that is the root of the graph. + string root_graph_node = 2; // (Required) Static display data for the pipeline. - DisplayData display_data = 7; + DisplayData display_data = 3; } // A generic node in a bipartite directed hierarchical graph. @@ -270,6 +311,7 @@ message ReadPayload { // The payload for the WindowInto transform. message WindowIntoPayload { + // (Required) The pipeline-scoped id for the FunctionSpec of the WindowFn. string fn_id = 1; }
