[
https://issues.apache.org/jira/browse/BEAM-3287?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16282196#comment-16282196
]
ASF GitHub Bot commented on BEAM-3287:
--------------------------------------
kennknowles closed pull request #4215: [BEAM-3287] Add Go SDK container image
URL: https://github.com/apache/beam/pull/4215
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/go/container/Dockerfile b/sdks/go/container/Dockerfile
new file mode 100644
index 00000000000..8fb9764274f
--- /dev/null
+++ b/sdks/go/container/Dockerfile
@@ -0,0 +1,30 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM debian:stretch
+MAINTAINER "Apache Beam <[email protected]>"
+
+RUN apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y \
+ ca-certificates \
+ && \
+ rm -rf /var/lib/apt/lists/*
+
+ADD target/linux_amd64/boot /opt/apache/beam/
+
+ENTRYPOINT ["/opt/apache/beam/boot"]
diff --git a/sdks/go/container/boot.go b/sdks/go/container/boot.go
new file mode 100644
index 00000000000..5259246a276
--- /dev/null
+++ b/sdks/go/container/boot.go
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "context"
+ "flag"
+ "io"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/apache/beam/sdks/go/pkg/beam/artifact"
+ "github.com/apache/beam/sdks/go/pkg/beam/provision"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/execx"
+ "github.com/apache/beam/sdks/go/pkg/beam/util/grpcx"
+)
+
+var (
+ // Contract: https://s.apache.org/beam-fn-api-container-contract.
+
+ id = flag.String("id", "", "Local identifier
(required).")
+ loggingEndpoint = flag.String("logging_endpoint", "", "Local logging
endpoint for FnHarness (required).")
+ artifactEndpoint = flag.String("artifact_endpoint", "", "Local
artifact endpoint for FnHarness (required).")
+ provisionEndpoint = flag.String("provision_endpoint", "", "Local
provision endpoint for FnHarness (required).")
+ controlEndpoint = flag.String("control_endpoint", "", "Local control
endpoint for FnHarness (required).")
+ semiPersistDir = flag.String("semi_persist_dir", "/tmp", "Local
semi-persistent directory (optional).")
+)
+
+func main() {
+ flag.Parse()
+ if *id == "" {
+ log.Fatal("No id provided.")
+ }
+ if *loggingEndpoint == "" {
+ log.Fatal("No logging endpoint provided.")
+ }
+ if *artifactEndpoint == "" {
+ log.Fatal("No artifact endpoint provided.")
+ }
+ if *provisionEndpoint == "" {
+ log.Fatal("No provision endpoint provided.")
+ }
+ if *controlEndpoint == "" {
+ log.Fatal("No control endpoint provided.")
+ }
+
+ log.Printf("Initializing Go harness: %v", strings.Join(os.Args, " "))
+
+ ctx := grpcx.WriteWorkerID(context.Background(), *id)
+
+ // (1) Obtain the pipeline options
+
+ info, err := provision.Info(ctx, *provisionEndpoint)
+ if err != nil {
+ log.Fatalf("Failed to obtain provisioning information: %v", err)
+ }
+ options, err := provision.ProtoToJSON(info.GetPipelineOptions())
+ if err != nil {
+ log.Fatalf("Failed to convert pipeline options: %v", err)
+ }
+
+ // (2) Retrieve the staged files.
+ //
+ // The Go SDK harness downloads the worker binary and invokes
+ // it. For now, we assume that the first (and only) package
+ // is the binary.
+
+ dir := filepath.Join(*semiPersistDir, "staged")
+ artifacts, err := artifact.Materialize(ctx, *artifactEndpoint, dir)
+ if err != nil {
+ log.Fatalf("Failed to retrieve staged files: %v", err)
+ }
+ if len(artifacts) == 0 {
+ log.Fatal("No binaries staged")
+ }
+
+ // (3) The persist dir may be on a noexec volume, so we must
+ // copy the binary to a different location to execute.
+
+ prog := filepath.Join("/bin", artifacts[0].Name)
+ if err := copyExe(filepath.Join(dir, artifacts[0].Name), prog); err !=
nil {
+ log.Fatalf("Failed to copy binary: %v", err)
+ }
+
+ args := []string{
+ "--worker=true",
+ "--id=" + *id,
+ "--logging_endpoint=" + *loggingEndpoint,
+ "--control_endpoint=" + *controlEndpoint,
+ "--semi_persist_dir=" + *semiPersistDir,
+ "--options=" + options,
+ }
+ log.Fatalf("User program exited: %v", execx.Execute(prog, args...))
+}
+
+func copyExe(from, to string) error {
+ src, err := os.Open(from)
+ if err != nil {
+ return err
+ }
+ defer src.Close()
+
+ dst, err := os.OpenFile(to, os.O_WRONLY|os.O_CREATE, 0755)
+ if err != nil {
+ return err
+ }
+
+ if _, err := io.Copy(dst, src); err != nil {
+ return err
+ }
+ return dst.Close()
+}
diff --git a/sdks/go/container/pom.xml b/sdks/go/container/pom.xml
new file mode 100644
index 00000000000..120e8194e42
--- /dev/null
+++ b/sdks/go/container/pom.xml
@@ -0,0 +1,154 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-go</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>beam-sdks-go-container</artifactId>
+
+ <packaging>pom</packaging>
+
+ <name>Apache Beam :: SDKs :: Go :: Container</name>
+
+ <properties>
+ <!-- Add full path directory structure for 'go get' compatibility -->
+ <go.source.base>${project.basedir}/target/src</go.source.base>
+
<go.source.dir>${go.source.base}/github.com/apache/beam/sdks/go</go.source.dir>
+ </properties>
+
+ <build>
+ <sourceDirectory>${go.source.base}</sourceDirectory>
+ <plugins>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-go-cmd-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+
<outputDirectory>${go.source.base}/github.com/apache/beam/cmd/boot</outputDirectory>
+ <resources>
+ <resource>
+ <directory>.</directory>
+ <includes>
+ <include>*.go</include>
+ </includes>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- CAVEAT: for latest shared files, run mvn install in sdks/go -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-dependency</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.beam</groupId>
+ <artifactId>beam-sdks-go</artifactId>
+ <version>${project.version}</version>
+ <type>zip</type>
+ <classifier>pkg-sources</classifier>
+ <overWrite>true</overWrite>
+ <outputDirectory>${go.source.dir}</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.igormaznitsa</groupId>
+ <artifactId>mvn-golang-wrapper</artifactId>
+ <executions>
+ <execution>
+ <id>go-get-imports</id>
+ <goals>
+ <goal>get</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>google.golang.org/grpc</package>
+ <package>golang.org/x/oauth2/google</package>
+ <package>google.golang.org/api/storage/v1</package>
+ </packages>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-build</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>github.com/apache/beam/cmd/boot</package>
+ </packages>
+ <resultName>boot</resultName>
+ </configuration>
+ </execution>
+ <execution>
+ <id>go-build-linux-amd64</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <packages>
+ <package>github.com/apache/beam/cmd/boot</package>
+ </packages>
+ <resultName>linux_amd64/boot</resultName>
+ <targetArch>amd64</targetArch>
+ <targetOs>linux</targetOs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>com.spotify</groupId>
+ <artifactId>dockerfile-maven-plugin</artifactId>
+ <configuration>
+ <repository>${docker-repository-root}/go</repository>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/sdks/go/pkg/beam/core/runtime/harness/init/init.go
b/sdks/go/pkg/beam/core/runtime/harness/init/init.go
index 1b9ed1adcb6..c99601da989 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/init/init.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/init/init.go
@@ -34,11 +34,14 @@ import (
)
var (
- // The below 4 flags implement the Fn API container contract. Subject
to change.
- worker = flag.Bool("worker", false, "Whether binary is running
in worker mode.")
+ // These flags handle the invocation by the container boot code.
+
+ worker = flag.Bool("worker", false, "Whether binary is running in
worker mode.")
+
+ id = flag.String("id", "", "Local identifier (required in
worker mode).")
loggingEndpoint = flag.String("logging_endpoint", "", "Local logging
gRPC endpoint (required in worker mode).")
controlEndpoint = flag.String("control_endpoint", "", "Local control
gRPC endpoint (required in worker mode).")
- persistDir = flag.String("persist_dir", "", "Local semi-persistent
directory (required in worker mode).")
+ semiPersistDir = flag.String("semi_persist_dir", "/tmp", "Local
semi-persistent directory (optional in worker mode).")
options = flag.String("options", "", "JSON-encoded pipeline
options (required in worker mode).")
)
diff --git a/sdks/go/pom.xml b/sdks/go/pom.xml
index 97c6c04fd19..6114254e41f 100644
--- a/sdks/go/pom.xml
+++ b/sdks/go/pom.xml
@@ -31,6 +31,10 @@
<name>Apache Beam :: SDKs :: Go</name>
+ <modules>
+ <module>container</module>
+ </modules>
+
<properties>
<!-- Add full path directory structure for 'go get' compatibility -->
<go.source.base>${project.basedir}/target/src</go.source.base>
@@ -81,6 +85,7 @@
<!-- export pkg/ sources as zip for inclusion elsewhere -->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
+ <inherited>false</inherited>
<executions>
<execution>
<id>export-go-pkg-sources</id>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Go SDK support for portable pipelines
> -------------------------------------
>
> Key: BEAM-3287
> URL: https://issues.apache.org/jira/browse/BEAM-3287
> Project: Beam
> Issue Type: Improvement
> Components: sdk-go
> Reporter: Henning Rohde
> Assignee: Henning Rohde
> Labels: portability
>
> The Go SDK should participate in the portability framework, incl. job
> submission w/ a docker container image.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
