[beam] branch master updated: [BEAM-11377] Fix credentials regression.
This is an automated email from the ASF dual-hosted git repository.
amyrvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new eeaacf4 [BEAM-11377] Fix credentials regression.
new 1ad764b Merge pull request #14190 from tysonjh/snapshot-fix-fix-fix
eeaacf4 is described below
commit eeaacf4b97d4a3c7ce2f4678ee5dda0bfe61cd16
Author: Tyson Hamilton
AuthorDate: Wed Mar 10 22:59:06 2021 +
[BEAM-11377] Fix credentials regression.
Previous 'fix' for retries broke credentials by executing on the
'ubuntu' tagged machine instead of 'beam'. This reverts that part of the
change.
---
.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
b/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
index 596e5d3..d31c354 100644
--- a/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
+++ b/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
@@ -24,7 +24,7 @@ job('beam_PostRelease_NightlySnapshot') {
description('Runs post release verification of the nightly snapshot.')
// Set common parameters. Timeout is longer, to avoid [BEAM-5774].
- commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 200,
true, 'ubuntu')
+ commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 200)
parameters {
stringParam('snapshot_version',
[beam] branch master updated: [BEAM-11796] Fix website build
This is an automated email from the ASF dual-hosted git repository.
amyrvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 6f3ed08 [BEAM-11796] Fix website build
new 1c5f3b4 Merge pull request #14038 from alanmyrvold/beamWebsite
6f3ed08 is described below
commit 6f3ed084f19b95aa5171fd85e2b32ff42f819aac
Author: Alan Myrvold
AuthorDate: Mon Feb 22 23:37:34 2021 +
[BEAM-11796] Fix website build
---
website/build.gradle | 2 ++
1 file changed, 2 insertions(+)
diff --git a/website/build.gradle b/website/build.gradle
index 6ea9e2a..5f6bd5a 100644
--- a/website/build.gradle
+++ b/website/build.gradle
@@ -272,6 +272,8 @@ task commitWebsite {
def latestCommit = grgit.log(maxCommits: 1)[0].abbreviatedId
shell "git fetch --force origin +asf-site:asf-site"
+shell "git stash"
+shell "git checkout asf-site"
git.checkout(branch: 'asf-site')
// Delete the previous content. These are asf-site branch paths.
[beam] branch master updated: [BEAM-10911] Grafana post-commit status board: job_name cells link to post-commit job in Jenkins
This is an automated email from the ASF dual-hosted git repository. amyrvold pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new af6ff6f [BEAM-10911] Grafana post-commit status board: job_name cells link to post-commit job in Jenkins new 88601d3 Merge pull request #13463 from dlu1000/testbranch af6ff6f is described below commit af6ff6f1d9d207cc7964b3fac27937a5fc4682fe Author: David Lu AuthorDate: Wed Dec 2 05:59:50 2020 -0800 [BEAM-10911] Grafana post-commit status board: job_name cells link to post-commit job in Jenkins --- .../metrics/grafana/dashboards/Post-Commits_status_dashboard.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/metrics/grafana/dashboards/Post-Commits_status_dashboard.json b/.test-infra/metrics/grafana/dashboards/Post-Commits_status_dashboard.json index 65fd62a..ea9aaee 100644 --- a/.test-infra/metrics/grafana/dashboards/Post-Commits_status_dashboard.json +++ b/.test-infra/metrics/grafana/dashboards/Post-Commits_status_dashboard.json @@ -49,7 +49,7 @@ "dateFormat": "-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, - "pattern": "/job_d*/", + "pattern": "/job_\\d/", "preserveFormat": false, "thresholds": [ "0.5",
[beam] branch master updated: [BEAM-11067] Update java license script to include source for GNU and Mozilla dependencies
This is an automated email from the ASF dual-hosted git repository.
amyrvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 098a36a [BEAM-11067] Update java license script to include source for
GNU and Mozilla dependencies
new f1ee42a Merge pull request #13117 from alanmyrvold/fixBEAM-11067
098a36a is described below
commit 098a36ac45f7b8d8b7cbca247f86cd272cb0e6f4
Author: Alan Myrvold
AuthorDate: Wed Oct 14 20:56:39 2020 +
[BEAM-11067] Update java license script to include source for GNU and
Mozilla dependencies
---
.../license_scripts/pull_licenses_java.py | 28 --
1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/sdks/java/container/license_scripts/pull_licenses_java.py
b/sdks/java/container/license_scripts/pull_licenses_java.py
index a587e30..acd1ac5 100644
--- a/sdks/java/container/license_scripts/pull_licenses_java.py
+++ b/sdks/java/container/license_scripts/pull_licenses_java.py
@@ -38,7 +38,7 @@ from tenacity import stop_after_attempt
from tenacity import wait_fixed
from urllib.request import urlopen, URLError, HTTPError
-SOURCE_CODE_REQUIRED_LICENSES = ['lgpl', 'glp', 'cddl', 'mpl']
+SOURCE_CODE_REQUIRED_LICENSES = ['lgpl', 'glp', 'cddl', 'mpl', 'gnu', 'mozilla
public license']
RETRY_NUM = 9
THREADS = 16
@@ -101,13 +101,22 @@ def pull_from_url(file_name, url, dep, no_list):
def pull_source_code(base_url, dir_name, dep):
# base_url example:
https://repo1.maven.org/maven2/org/mortbay/jetty/jsp-2.1/6.1.14/
-soup = BeautifulSoup(urlopen(base_url).read(), "html.parser")
+try:
+ soup = BeautifulSoup(urlopen(base_url).read(), "html.parser")
+except:
+ logging.error('Error reading source base from
{base_url}'.format(base_url=base_url))
+ raise
+source_count = 0
for href in (a["href"] for a in soup.select("a[href]")):
if href.endswith(
-'.jar') and not 'javadoc' in href: # download jar file only
+'.jar') and 'sources.jar' in href: # download sources jar
file only
file_name = dir_name + '/' + href
url = base_url + '/' + href
+logging.debug('Pulling source from {url}'.format(url=url))
pull_from_url(file_name, url, dep, incorrect_source_url)
+source_count = source_count + 1
+if source_count == 0:
+ raise RuntimeError('No source found at
{base_url}'.format(base_url=base_url))
@retry(reraise=True, stop=stop_after_attempt(3))
@@ -140,17 +149,22 @@ def execute(dep):
}
'''
-name = dep['moduleName'].split(':')[1].lower()
+name = dep['moduleName'].split(':')[1]
version = dep['moduleVersion']
name_version = name + '-' + version
+# javac is not a runtime dependency
+if name == 'javac':
+ logging.debug('Skipping', name_version)
+ return
+# skip self dependencies
+if dep['moduleName'].lower().startswith('beam'):
+ logging.debug('Skipping', name_version)
+ return
dir_name = '{license_dir}/{name_version}.jar'.format(
license_dir=license_dir, name_version=name_version)
# if auto pulled, directory is existing at {license_dir}
if not os.path.isdir(dir_name):
-# skip self dependencies
-if dep['moduleName'].startswith('beam'):
-logging.debug('Skippig', name_version)
os.mkdir(dir_name)
# pull license
try:
[beam] branch master updated: [BEAM-9136] Add python dependency license CSV for license URL and type
This is an automated email from the ASF dual-hosted git repository.
amyrvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 8fe5805 [BEAM-9136] Add python dependency license CSV for license URL
and type
new 9d2fce7 Merge pull request #12879 from alanmyrvold/py-license-list
8fe5805 is described below
commit 8fe5805454d957d40127efee5236461712642dc3
Author: Alan Myrvold
AuthorDate: Fri Sep 18 23:54:44 2020 +
[BEAM-9136] Add python dependency license CSV for license URL and type
---
sdks/python/container/Dockerfile | 6 ++-
.../container/license_scripts/pull_licenses_py.py | 50 --
2 files changed, 52 insertions(+), 4 deletions(-)
diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile
index a7c498b..9d1d3a8 100644
--- a/sdks/python/container/Dockerfile
+++ b/sdks/python/container/Dockerfile
@@ -45,7 +45,8 @@ RUN \
# Check that the fast implementation of protobuf is used.
python -c "from google.protobuf.internal import api_implementation; assert
api_implementation._default_implementation_type == 'cpp'; print ('Verified fast
protobuf used.')" && \
# Remove pip cache.
-rm -rf /root/.cache/pip
+rm -rf /root/.cache/pip && \
+rm -rf /tmp/base_image_requirements.txt
# Configure ccache prior to installing the SDK.
RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
@@ -76,6 +77,9 @@ RUN if [ "$pull_licenses" = "true" ] ; then \
rm -rf /opt/apache/beam/third_party_licenses ; \
fi
+# Remove license scripts
+RUN rm -rf /tmp/license_scripts
+
# Log complete list of what exact packages and versions are installed.
RUN pip freeze --all
# Make sure there are no conflicting dependencies.
diff --git a/sdks/python/container/license_scripts/pull_licenses_py.py
b/sdks/python/container/license_scripts/pull_licenses_py.py
index c987015..75f629d 100644
--- a/sdks/python/container/license_scripts/pull_licenses_py.py
+++ b/sdks/python/container/license_scripts/pull_licenses_py.py
@@ -19,6 +19,7 @@
A script to pull licenses for Python.
The script is executed within Docker.
"""
+import csv
import json
import logging
import os
@@ -30,6 +31,9 @@ import traceback
import yaml
from future.moves.urllib.request import urlopen
+from future.moves.urllib.parse import urlparse
+from future.moves.urllib.parse import urljoin
+
from tenacity import retry
from tenacity import stop_after_attempt
from tenacity import wait_exponential
@@ -42,7 +46,7 @@ def run_bash_command(command):
def run_pip_licenses():
- command = 'pip-licenses --with-license-file --format=json'
+ command = 'pip-licenses --with-license-file --with-urls --from=mixed
--ignore apache-beam --format=json'
dependencies = run_bash_command(command)
return json.loads(dependencies)
@@ -53,7 +57,7 @@ def copy_license_files(dep):
if source_license_file.lower() == 'unknown':
return False
name = dep['Name'].lower()
- dest_dir = '/'.join([LICENSE_DIR, name])
+ dest_dir = os.path.join(LICENSE_DIR, name)
try:
os.mkdir(dest_dir)
shutil.copy(source_license_file, dest_dir + '/LICENSE')
@@ -84,7 +88,7 @@ def pull_from_url(dep, configs):
'''
if dep in configs:
config = configs[dep]
-dest_dir = '/'.join([LICENSE_DIR, dep])
+dest_dir = os.path.join(LICENSE_DIR, dep)
cur_temp_dir = tempfile.mkdtemp()
try:
@@ -116,6 +120,43 @@ def pull_from_url(dep, configs):
shutil.rmtree(cur_temp_dir)
+def license_url(name, project_url, dep_config):
+ '''
+ Gets the license URL for a dependency, either from the parsed yaml or,
+ if it is github, by looking for a license file in the repo.
+ '''
+ configs = dep_config['pip_dependencies']
+ if name.lower() in configs:
+return configs[name.lower()]['license']
+ p = urlparse(project_url)
+ if p.netloc != "github.com":
+return project_url
+ raw = "https://raw.githubusercontent.com;
+ path = p.path
+ if not path.endswith("/"):
+path = path + "/"
+ for license in ("LICENSE", "LICENSE.txt", "LICENSE.md", "LICENSE.rst",
"COPYING"):
+try:
+ url = raw + urljoin(path,"master/"+license)
+ with urlopen(url) as a:
+if a.getcode() == 200:
+ return url
+except:
+ pass
+ return project_url
+
+
+def save_license_list(csv_filename, dependencies, dep_config):
+ '''
+ Save the names, URLs, and license type for python dependency licenses in a
CSV file.
+ '''
+ with open(csv_filename, mode='w') as f:
+writer = csv.writer(f)
+for dep in dependencies:
+ url = license_url(dep['Name'], dep['URL'], dep_config)
+ writer.writerow([dep['Name'], url, dep['License']])
+
+
if __name__ == "__main__":
[beam] branch master updated: [BEAM-10868] Fix build (docker run) concurrency issue when building golang license dependencies
This is an automated email from the ASF dual-hosted git repository.
amyrvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 803efdd [BEAM-10868] Fix build (docker run) concurrency issue when
building golang license dependencies
new 856cc22 Merge pull request #12829 from alanmyrvold/flaky-test
803efdd is described below
commit 803efdd931699c24e60350e5ccf6e54482f5916f
Author: Alan Myrvold
AuthorDate: Fri Sep 11 17:29:37 2020 +
[BEAM-10868] Fix build (docker run) concurrency issue when building golang
license dependencies
---
release/go-licenses/common.gradle | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/release/go-licenses/common.gradle
b/release/go-licenses/common.gradle
index 751b8d9..172bdfb 100644
--- a/release/go-licenses/common.gradle
+++ b/release/go-licenses/common.gradle
@@ -30,7 +30,8 @@ docker {
mkdir "${licenseOutput}"
dockerRun {
- name "license-container-${sdkName}"
+ // Add a random number to the running container name to avoid collisions.
+ name "license-container-${sdkName}-" + (new Random().nextInt())
image "golicenses-${sdkName}:latest"
volumes "${licenseOutput}": '/output'
daemonize false
@@ -41,5 +42,8 @@ tasks.dockerRun.dependsOn 'docker'
task createLicenses {
outputs.cacheIf { true }
outputs.file("$buildDir/output/licenses/list.csv")
- dependsOn 'dockerRun'
+ // Aggressively cache, to avoid slowness and docker run collisions.
+ if (!file("$buildDir/output/licenses/list.csv").exists()) {
+dependsOn 'dockerRun'
+ }
}
[beam] branch master updated: [BEAM-10049] Add licenses for go dependencies in python, java, and go SDK containers
This is an automated email from the ASF dual-hosted git repository.
amyrvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 081ad8c [BEAM-10049] Add licenses for go dependencies in python,
java, and go SDK containers
new 209bc21 Merge pull request #12650 from alanmyrvold/licensesGo
081ad8c is described below
commit 081ad8c33fb2bd53df6cff39f09eed7ce804ce05
Author: Alan Myrvold
AuthorDate: Thu Aug 20 22:07:08 2020 +
[BEAM-10049] Add licenses for go dependencies in python, java, and go SDK
containers
---
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 3 ++
.../container => release/go-licenses}/Dockerfile | 21 +++---
release/go-licenses/common.gradle | 45 ++
release/go-licenses/get-licenses.sh| 26 +
release/go-licenses/go/build.gradle| 25
release/go-licenses/java/build.gradle | 25
release/go-licenses/py/build.gradle| 25
sdks/go/container/Dockerfile | 12 ++
sdks/go/container/build.gradle | 13 ++-
sdks/java/container/Dockerfile | 13 +++
sdks/java/container/Dockerfile-java11 | 12 ++
sdks/java/container/build.gradle | 9 +
sdks/python/container/Dockerfile | 12 ++
sdks/python/container/common.gradle| 9 +
.../container/license_scripts/pull_licenses_py.py | 1 -
settings.gradle| 3 ++
16 files changed, 237 insertions(+), 17 deletions(-)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index ffe1e3c..3133c4f 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -1497,6 +1497,9 @@ class BeamModulePlugin implements Plugin {
project.tasks.dockerPrepare.dependsOn project.tasks.copyLicenses
}
+project.ext.applyDockerRunNature = {
+ project.apply plugin: "com.palantir.docker-run"
+}
/**
***/
project.ext.applyGroovyNature = {
diff --git a/sdks/go/container/Dockerfile b/release/go-licenses/Dockerfile
similarity index 71%
copy from sdks/go/container/Dockerfile
copy to release/go-licenses/Dockerfile
index 770d4eb..43e7870 100644
--- a/sdks/go/container/Dockerfile
+++ b/release/go-licenses/Dockerfile
@@ -16,18 +16,9 @@
# limitations under the License.
###
-FROM debian:buster
-MAINTAINER "Apache Beam "
-
-RUN apt-get update && \
-DEBIAN_FRONTEND=noninteractive apt-get install -y \
-ca-certificates \
-&& \
-rm -rf /var/lib/apt/lists/*
-
-ADD target/linux_amd64/boot /opt/apache/beam/
-
-COPY target/LICENSE /opt/apache/beam/
-COPY target/NOTICE /opt/apache/beam/
-
-ENTRYPOINT ["/opt/apache/beam/boot"]
+FROM golang:1.15.0-buster
+RUN go get github.com/google/go-licenses
+COPY get-licenses.sh /opt/apache/beam/
+ARG sdk_location
+ENV sdk_location=$sdk_location
+ENTRYPOINT /opt/apache/beam/get-licenses.sh
diff --git a/release/go-licenses/common.gradle
b/release/go-licenses/common.gradle
new file mode 100644
index 000..751b8d9
--- /dev/null
+++ b/release/go-licenses/common.gradle
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+def licenseOutput = "${buildDir}/output"
+
+docker {
+ name "golicenses-${sdkName}"
+ dockerfile file('../Dockerfile')
+ files '../get-licenses.sh'
+ noCache false
+ buildArgs(['sdk_location':
"github.com/apache/beam/sdks/${sdkName}/container"])
+}
+
+// The mkdir happens at configuration time to allow dockerRun volume to be
declared.
+mkdir "${licenseOutput}"
+
+dockerRun {
[beam] branch master updated: [BEAM-9261] Add LICENSE and NOTICE to Docker images
This is an automated email from the ASF dual-hosted git repository.
amyrvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e0cd0d4 [BEAM-9261] Add LICENSE and NOTICE to Docker images
new d976ddc Merge pull request #10788 from alanmyrvold/addLicenseToDocker
e0cd0d4 is described below
commit e0cd0d43e33c869c60d5811a482a2f95c316c1d9
Author: Alan Myrvold
AuthorDate: Thu Feb 6 09:16:10 2020 -0800
[BEAM-9261] Add LICENSE and NOTICE to Docker images
---
.../src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 7 +++
runners/flink/job-server-container/Dockerfile | 3 +++
runners/spark/job-server/container/Dockerfile | 4
sdks/go/container/Dockerfile | 3 +++
sdks/java/container/Dockerfile | 3 +++
sdks/python/container/Dockerfile | 3 +++
6 files changed, 23 insertions(+)
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 170c86b..68f2f9c 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -31,6 +31,7 @@ import org.gradle.api.artifacts.ProjectDependency
import org.gradle.api.file.FileTree
import org.gradle.api.plugins.quality.Checkstyle
import org.gradle.api.publish.maven.MavenPublication
+import org.gradle.api.tasks.Copy
import org.gradle.api.tasks.Exec
import org.gradle.api.tasks.JavaExec
import org.gradle.api.tasks.bundling.Jar
@@ -1390,6 +1391,12 @@ class BeamModulePlugin implements Plugin {
project.ext.applyDockerNature = {
project.apply plugin: "com.palantir.docker"
project.docker { noCache true }
+ project.tasks.create(name: "copyLicenses", type: Copy) {
+from "${project.rootProject.projectDir}/LICENSE"
+from "${project.rootProject.projectDir}/NOTICE"
+into "build/target"
+ }
+ project.tasks.dockerPrepare.dependsOn project.tasks.copyLicenses
}
/**
***/
diff --git a/runners/flink/job-server-container/Dockerfile
b/runners/flink/job-server-container/Dockerfile
index 0381131..c5a81ec 100644
--- a/runners/flink/job-server-container/Dockerfile
+++ b/runners/flink/job-server-container/Dockerfile
@@ -24,5 +24,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get
install -y libltdl7
ADD beam-runners-flink-job-server.jar /opt/apache/beam/jars/
ADD flink-job-server.sh /opt/apache/beam/
+COPY target/LICENSE /opt/apache/beam/
+COPY target/NOTICE /opt/apache/beam/
+
WORKDIR /opt/apache/beam
ENTRYPOINT ["./flink-job-server.sh"]
diff --git a/runners/spark/job-server/container/Dockerfile
b/runners/spark/job-server/container/Dockerfile
index 11a5f0e..ec4a123 100644
--- a/runners/spark/job-server/container/Dockerfile
+++ b/runners/spark/job-server/container/Dockerfile
@@ -25,4 +25,8 @@ ADD beam-runners-spark-job-server.jar /opt/apache/beam/jars/
ADD spark-job-server.sh /opt/apache/beam/
WORKDIR /opt/apache/beam
+
+COPY target/LICENSE /opt/apache/beam/
+COPY target/NOTICE /opt/apache/beam/
+
ENTRYPOINT ["./spark-job-server.sh"]
diff --git a/sdks/go/container/Dockerfile b/sdks/go/container/Dockerfile
index 8fb9764..52c6eae 100644
--- a/sdks/go/container/Dockerfile
+++ b/sdks/go/container/Dockerfile
@@ -27,4 +27,7 @@ RUN apt-get update && \
ADD target/linux_amd64/boot /opt/apache/beam/
+COPY target/LICENSE /opt/apache/beam/
+COPY target/NOTICE /opt/apache/beam/
+
ENTRYPOINT ["/opt/apache/beam/boot"]
diff --git a/sdks/java/container/Dockerfile b/sdks/java/container/Dockerfile
index af5c31e..ba8b1a8 100644
--- a/sdks/java/container/Dockerfile
+++ b/sdks/java/container/Dockerfile
@@ -30,4 +30,7 @@ ADD target/kafka-clients.jar /opt/apache/beam/jars/
ADD target/linux_amd64/boot /opt/apache/beam/
+COPY target/LICENSE /opt/apache/beam/
+COPY target/NOTICE /opt/apache/beam/
+
ENTRYPOINT ["/opt/apache/beam/boot"]
diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile
index 4e6b62c..a10cd72 100644
--- a/sdks/python/container/Dockerfile
+++ b/sdks/python/container/Dockerfile
@@ -60,4 +60,7 @@ RUN pip freeze --all
ADD target/launcher/linux_amd64/boot /opt/apache/beam/
+COPY target/LICENSE /opt/apache/beam/
+COPY target/NOTICE /opt/apache/beam/
+
ENTRYPOINT ["/opt/apache/beam/boot"]
[beam] branch master updated: [BEAM-8575] Fix window assignment idempotency tests non-deterministic elements order
This is an automated email from the ASF dual-hosted git repository.
amyrvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 312535e [BEAM-8575] Fix window assignment idempotency tests
non-deterministic elements order
new 9009dd1 Merge pull request #10111 from
liumomo315/fix_window_idempotency
312535e is described below
commit 312535e4545e5bbffc76032b7f2cd3da6d287559
Author: Wenjia Liu
AuthorDate: Thu Nov 14 11:49:20 2019 -0800
[BEAM-8575] Fix window assignment idempotency tests non-deterministic
elements order
---
sdks/python/apache_beam/transforms/window_test.py | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/sdks/python/apache_beam/transforms/window_test.py
b/sdks/python/apache_beam/transforms/window_test.py
index 0d5e14f..3c45d83 100644
--- a/sdks/python/apache_beam/transforms/window_test.py
+++ b/sdks/python/apache_beam/transforms/window_test.py
@@ -286,21 +286,21 @@ class WindowTest(unittest.TestCase):
@attr('ValidatesRunner')
def test_window_assignment_idempotency(self):
with TestPipeline() as p:
- pcoll = self.timestamped_key_values(p, 'key', 0, 1, 2, 3, 4)
+ pcoll = self.timestamped_key_values(p, 'key', 0, 2, 4)
result = (pcoll
| 'window' >> WindowInto(FixedWindows(2))
| 'same window' >> WindowInto(FixedWindows(2))
| 'same window again' >> WindowInto(FixedWindows(2))
| GroupByKey())
- assert_that(result, equal_to([('key', [0, 1]),
-('key', [2, 3]),
+ assert_that(result, equal_to([('key', [0]),
+('key', [2]),
('key', [4])]))
@attr('ValidatesRunner')
def test_window_assignment_through_multiple_gbk_idempotency(self):
with TestPipeline() as p:
- pcoll = self.timestamped_key_values(p, 'key', 0, 1, 2, 3, 4)
+ pcoll = self.timestamped_key_values(p, 'key', 0, 2, 4)
result = (pcoll
| 'window' >> WindowInto(FixedWindows(2))
| 'gbk' >> GroupByKey()
@@ -309,8 +309,8 @@ class WindowTest(unittest.TestCase):
| 'same window again' >> WindowInto(FixedWindows(2))
| 'gbk again' >> GroupByKey())
- assert_that(result, equal_to([('key', [[[0, 1]]]),
-('key', [[[2, 3]]]),
+ assert_that(result, equal_to([('key', [[[0]]]),
+('key', [[[2]]]),
('key', [[[4]]])]))
class RunnerApiTest(unittest.TestCase):
