This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 3e2e630 [BEAM-12095] Fix Spark job server path.
new 312c6eb Merge pull request #14429 from ibzib/BEAM-12095
3e2e630 is described below
commit 3e2e630fa1f93e182d324c43ba49c4b4e1862031
Author: Kyle Weaver <kcwea...@google.com>
AuthorDate: Mon Apr 5 09:53:38 2021 -0700
[BEAM-12095] Fix Spark job server path.
---
sdks/python/apache_beam/runners/portability/job_server.py | 5 +++--
sdks/python/apache_beam/runners/portability/spark_runner.py | 4 +++-
sdks/python/apache_beam/utils/subprocess_server.py | 10 ++++++++--
3 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/sdks/python/apache_beam/runners/portability/job_server.py
b/sdks/python/apache_beam/runners/portability/job_server.py
index dfc8802..2d581c4 100644
--- a/sdks/python/apache_beam/runners/portability/job_server.py
+++ b/sdks/python/apache_beam/runners/portability/job_server.py
@@ -140,8 +140,9 @@ class JavaJarJobServer(SubprocessJobServer):
raise NotImplementedError(type(self))
@staticmethod
- def path_to_beam_jar(gradle_target):
- return subprocess_server.JavaJarServer.path_to_beam_jar(gradle_target)
+ def path_to_beam_jar(gradle_target, artifact_id=None):
+ return subprocess_server.JavaJarServer.path_to_beam_jar(
+ gradle_target, artifact_id=artifact_id)
@staticmethod
def local_jar(url):
diff --git a/sdks/python/apache_beam/runners/portability/spark_runner.py
b/sdks/python/apache_beam/runners/portability/spark_runner.py
index ec7046c..99337b0 100644
--- a/sdks/python/apache_beam/runners/portability/spark_runner.py
+++ b/sdks/python/apache_beam/runners/portability/spark_runner.py
@@ -82,7 +82,9 @@ class SparkJarJobServer(job_server.JavaJarJobServer):
self._jar)
return self._jar
else:
- return self.path_to_beam_jar(':runners:spark:2:job-server:shadowJar')
+ return self.path_to_beam_jar(
+ ':runners:spark:2:job-server:shadowJar',
+ artifact_id='beam-runners-spark-job-server')
def java_arguments(
self, job_port, artifact_port, expansion_port, artifacts_dir):
diff --git a/sdks/python/apache_beam/utils/subprocess_server.py
b/sdks/python/apache_beam/utils/subprocess_server.py
index 7ff4261..d7af0b4 100644
--- a/sdks/python/apache_beam/utils/subprocess_server.py
+++ b/sdks/python/apache_beam/utils/subprocess_server.py
@@ -205,12 +205,18 @@ class JavaJarServer(SubprocessServer):
])
@classmethod
- def path_to_beam_jar(cls, gradle_target, appendix=None,
version=beam_version):
+ def path_to_beam_jar(
+ cls,
+ gradle_target,
+ appendix=None,
+ version=beam_version,
+ artifact_id=None):
if gradle_target in cls._BEAM_SERVICES.replacements:
return cls._BEAM_SERVICES.replacements[gradle_target]
gradle_package = gradle_target.strip(':').rsplit(':', 1)[0]
- artifact_id = 'beam-' + gradle_package.replace(':', '-')
+ if not artifact_id:
+ artifact_id = 'beam-' + gradle_package.replace(':', '-')
project_root = os.path.sep.join(
os.path.abspath(__file__).split(os.path.sep)[:-5])
local_path = os.path.join(
