This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 1981cb0f6cc [#27839] Write PipelineOptions to a file instead of an environment variable. (#27842) 1981cb0f6cc is described below commit 1981cb0f6cc62ee3a679e195d3bdd83da03ba6e7 Author: Robert Burke <lostl...@users.noreply.github.com> AuthorDate: Tue Aug 15 18:08:51 2023 -0700 [#27839] Write PipelineOptions to a file instead of an environment variable. (#27842) * remove use of deprecated io/ioutil package. * [#27839] Write pipeline options to a local file for SDK use. * finish comment * Use guave 32.1.2 * fix bad merges * further merge fixes * Fix write error handling. --------- Co-authored-by: lostluck <13907733+lostl...@users.noreply.github.com> Co-authored-by: Yi Hu <ya...@google.com> --- sdks/java/container/boot.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/sdks/java/container/boot.go b/sdks/java/container/boot.go index 61d5a6a3faa..0e39d907f07 100644 --- a/sdks/java/container/boot.go +++ b/sdks/java/container/boot.go @@ -22,7 +22,6 @@ import ( "encoding/json" "flag" "fmt" - "io/ioutil" "log" "os" "path/filepath" @@ -125,7 +124,9 @@ func main() { // (3) Invoke the Java harness, preserving artifact ordering in classpath. os.Setenv("HARNESS_ID", *id) - os.Setenv("PIPELINE_OPTIONS", options) + if err := makePipelineOptionsFile(options); err != nil { + logger.Fatalf(ctx, "Failed to load pipeline options to worker: %v", err) + } os.Setenv("LOGGING_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(&pipepb.ApiServiceDescriptor{Url: *loggingEndpoint})) os.Setenv("CONTROL_API_SERVICE_DESCRIPTOR", proto.MarshalTextString(&pipepb.ApiServiceDescriptor{Url: *controlEndpoint})) os.Setenv("RUNNER_CAPABILITIES", strings.Join(info.GetRunnerCapabilities(), " ")) @@ -245,6 +246,22 @@ func main() { logger.Fatalf(ctx, "Java exited: %v", execx.Execute("java", args...)) } +// makePipelineOptionsFile writes the pipeline options to a file. +// Assumes the options string is JSON formatted. +func makePipelineOptionsFile(options string) error { + fn := "pipeline_options.json" + f, err := os.Create(fn) + if err != nil { + return fmt.Errorf("unable to create %v: %w", fn, err) + } + defer f.Close() + if _, err := f.WriteString(options); err != nil { + return fmt.Errorf("error writing %v: %w", f.Name(), err) + } + os.Setenv("PIPELINE_OPTIONS_FILE", f.Name()) + return nil +} + // heapSizeLimit returns 80% of the runner limit, if provided. If not provided, // it returns 70% of the physical memory on the machine. If it cannot determine // that value, it returns 1GB. This is an imperfect heuristic. It aims to @@ -327,7 +344,7 @@ func LoadMetaOptions(ctx context.Context, logger *tools.Logger, dir string) ([]* return nil } - content, err := ioutil.ReadFile(path) + content, err := os.ReadFile(path) if err != nil { return err }