tvalentyn commented on code in PR #25970:
URL: https://github.com/apache/beam/pull/25970#discussion_r1153769539
##########
buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy:
##########
@@ -2785,15 +2785,70 @@ class BeamModulePlugin implements Plugin<Project> {
// distribution tarball generated by :sdks:python:sdist.
project.configurations { distTarBall }
+ // Create a task to install Beam python SDK locally before running tests
on
+ // Google cloud platform. The task depends on ':sdks:python:sdist'
project
+ // for the tarball file.
+ // For compatible systems (Linux amd64), a 'sdistReuseWheel' system
property
+ // can be provided to re-use the wheel built in this task (by configuring
+ // `--sdk_location` pipeine option in test tasks).
def installGcpTest = project.tasks.register('installGcpTest') {
dependsOn setupVirtualenv
dependsOn ':sdks:python:sdist'
+
+ // if host system wheel compatible with dataflow runner
+ def compatibleWithDataflow =
("amd64".equalsIgnoreCase(System.getProperty("os.arch")) &&
+ "linux".equalsIgnoreCase(System.getProperty("os.name")))
+ if (!compatibleWithDataflow && project.hasProperty('sdistReuseWheel'))
{
+ throw new GradleException('-PsdistReuseWheel is set for the task but
the ' +
+ 'host system is not compatible with Dataflow worker container
image.')
+ }
+
+ // Set sdistFiles project ext at execution time as the path to the
+ // generated installable Python SDK package. If project property
+ // sdistUseWheel is set, targets to wheel, otherwise a tarball.
+ project.ext.sdistFiles = project.files()
+
doLast {
- def distTarBall = "${pythonRootDir}/build/apache-beam.tar.gz"
+ def packageFile = "${pythonRootDir}/build/apache-beam.tar.gz"
+ if (compatibleWithDataflow) {
+ // build wheel in separate folder to avoid racing conditions
+ project.copy {
+ from project.tarTree(project.resources.gzip(packageFile))
+ into project.buildDir
+ }
+ def srcDirs = project.files()
+ project.buildDir.eachDirMatch({it.startsWith('apache-beam')},
{srcDirs.from it})
+ def srcDir = srcDirs.singleFile
+ project.exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate && pip install
'Cython<1' " +
+ "&& cd ${srcDir} && python setup.py -q sdist bdist_wheel
--dist-dir ${project.buildDir}"
+ }
+ def collection = project.fileTree(project.buildDir){
+ include "**/*${project.ext.pythonVersion.replace('.', '')}*.whl"
+ exclude 'srcs/**'
+ }
+ def packageFilename = collection.singleFile.getName()
+ // rename to the suffix accepted by sdks/python/container/boot.go.
+ def renamed = packageFilename.replace(
Review Comment:
ok I see. Can we build the wheel with a configuration setting way where we
wouldn't have to rename it?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
