This is an automated email from the ASF dual-hosted git repository.
tvalentyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 06588745182 [yaml] disable javascript mapping for python >=3.12
(#30843)
06588745182 is described below
commit 065887451827f5e33d63644fe222dc933fba1147
Author: Jeff Kinard <[email protected]>
AuthorDate: Tue Apr 9 19:54:15 2024 -0400
[yaml] disable javascript mapping for python >=3.12 (#30843)
* [yaml] disable javascript mapping for python >=3.12
Signed-off-by: Jeffrey Kinard <[email protected]>
* use environmental marker for js2py
Signed-off-by: Jeffrey Kinard <[email protected]>
* Update sdks/python/setup.py
---------
Signed-off-by: Jeffrey Kinard <[email protected]>
Co-authored-by: tvalentyn <[email protected]>
---
sdks/python/apache_beam/yaml/yaml_mapping.py | 26 ++++++++++++++++++++------
sdks/python/apache_beam/yaml/yaml_udf_test.py | 10 ++++++++++
sdks/python/setup.py | 3 ++-
3 files changed, 32 insertions(+), 7 deletions(-)
diff --git a/sdks/python/apache_beam/yaml/yaml_mapping.py
b/sdks/python/apache_beam/yaml/yaml_mapping.py
index 4839728dd88..32095fe39f2 100644
--- a/sdks/python/apache_beam/yaml/yaml_mapping.py
+++ b/sdks/python/apache_beam/yaml/yaml_mapping.py
@@ -31,11 +31,6 @@ from typing import Optional
from typing import TypeVar
from typing import Union
-import js2py
-from js2py import base
-from js2py.constructors import jsdate
-from js2py.internals import simplex
-
import apache_beam as beam
from apache_beam.io.filesystems import FileSystems
from apache_beam.portability.api import schema_pb2
@@ -52,6 +47,14 @@ from apache_beam.yaml import options
from apache_beam.yaml import yaml_provider
from apache_beam.yaml.yaml_provider import dicts_to_rows
+# Import js2py package if it exists
+try:
+ import js2py
+ from js2py.base import JsObjectWrapper
+except ImportError:
+ js2py = None
+ JsObjectWrapper = object
+
def normalize_mapping(spec):
"""
@@ -87,7 +90,7 @@ def _check_mapping_arguments(
# js2py's JsObjectWrapper object has a self-referencing __dict__ property
# that cannot be pickled without implementing the __getstate__ and
# __setstate__ methods.
-class _CustomJsObjectWrapper(js2py.base.JsObjectWrapper):
+class _CustomJsObjectWrapper(JsObjectWrapper):
def __init__(self, js_obj):
super().__init__(js_obj.__dict__['_obj'])
@@ -116,6 +119,17 @@ def py_value_to_js_dict(py_value):
def _expand_javascript_mapping_func(
original_fields, expression=None, callable=None, path=None, name=None):
+ # Check for installed js2py package
+ if js2py is None:
+ raise ValueError(
+ "Javascript mapping functions are not supported on"
+ " Python 3.12 or later.")
+
+ # import remaining js2py objects
+ from js2py import base
+ from js2py.constructors import jsdate
+ from js2py.internals import simplex
+
js_array_type = (
base.PyJsArray,
base.PyJsArrayBuffer,
diff --git a/sdks/python/apache_beam/yaml/yaml_udf_test.py
b/sdks/python/apache_beam/yaml/yaml_udf_test.py
index 5f5ee1147de..c26d8ec92dd 100644
--- a/sdks/python/apache_beam/yaml/yaml_udf_test.py
+++ b/sdks/python/apache_beam/yaml/yaml_udf_test.py
@@ -29,6 +29,12 @@ from apache_beam.yaml.yaml_mapping import py_value_to_js_dict
from apache_beam.yaml.yaml_provider import dicts_to_rows
from apache_beam.yaml.yaml_transform import YamlTransform
+try:
+ import js2py
+except ImportError:
+ js2py = None
+ logging.warning('js2py is not installed; some tests will be skipped.')
+
def AsRows():
return beam.Map(
@@ -55,6 +61,7 @@ class YamlUDFMappingTest(unittest.TestCase):
def tearDown(self):
shutil.rmtree(self.tmpdir)
+ @unittest.skipIf(js2py is None, 'js2py not installed.')
def test_map_to_fields_filter_inline_js(self):
with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions(
pickle_library='cloudpickle', yaml_experimental_features=['javascript'
@@ -125,6 +132,7 @@ class YamlUDFMappingTest(unittest.TestCase):
beam.Row(label='389ax', conductor=390, sum=24),
]))
+ @unittest.skipIf(js2py is None, 'js2py not installed.')
def test_filter_inline_js(self):
with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions(
pickle_library='cloudpickle', yaml_experimental_features=['javascript'
@@ -179,6 +187,7 @@ class YamlUDFMappingTest(unittest.TestCase):
row=beam.Row(rank=2, values=[7, 8, 9])),
]))
+ @unittest.skipIf(js2py is None, 'js2py not installed.')
def test_filter_expression_js(self):
with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions(
pickle_library='cloudpickle', yaml_experimental_features=['javascript'
@@ -222,6 +231,7 @@ class YamlUDFMappingTest(unittest.TestCase):
row=beam.Row(rank=0, values=[1, 2, 3])),
]))
+ @unittest.skipIf(js2py is None, 'js2py not installed.')
def test_filter_inline_js_file(self):
data = '''
function f(x) {
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 2975f16d40d..400083e0a0a 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -368,7 +368,8 @@ if __name__ == '__main__':
'grpcio>=1.33.1,!=1.48.0,<2',
'hdfs>=2.1.0,<3.0.0',
'httplib2>=0.8,<0.23.0',
- 'js2py>=0.74,<1',
+ # https://github.com/PiotrDabkowski/Js2Py/issues/317
+ 'js2py>=0.74,<1; python_version<"3.12"',
'jsonschema>=4.0.0,<5.0.0',
'jsonpickle>=3.0.0,<4.0.0',
# numpy can have breaking changes in minor versions.
