This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new dbf9e500e56 [Python] Return every group from Regex.find_all with
Regex.ALL (#40399)
dbf9e500e56 is described below
commit dbf9e500e5626a6409bcfb58be121cec31b34a6d
Author: Anish Mehta <[email protected]>
AuthorDate: Mon Oct 12 23:44:18 2026 +0530
[Python] Return every group from Regex.find_all with Regex.ALL (#40399)
* [Python] Return every group from Regex.find_all with Regex.ALL
* Update sdks/python/apache_beam/transforms/util.py
Co-authored-by: Danny McCormick <[email protected]>
* Test outputEmpty=False with two groups and with empty matches
* Gate Regex.ALL outputEmpty on the full match and note the change in
CHANGES.md
* Update sdks/python/apache_beam/transforms/util.py
Co-authored-by: Danny McCormick <[email protected]>
* Apply suggestion from @damccorm
Co-authored-by: Danny McCormick <[email protected]>
* Drop the leftover outputEmpty line and fix the docstring indent
* Update CHANGES.md
Co-authored-by: Danny McCormick <[email protected]>
---------
Co-authored-by: Danny McCormick <[email protected]>
---
CHANGES.md | 1 +
sdks/python/apache_beam/transforms/util.py | 13 ++++++-----
sdks/python/apache_beam/transforms/util_test.py | 29 ++++++++++++++++++++++++-
3 files changed, 36 insertions(+), 7 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 0525ff92792..b0767fbe9d6 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -81,6 +81,7 @@
* (Java) JmsIO migrated to `jakarta.jms` (JMS 3.1) and ActiveMQ 6.2.5. User
code implementing `JmsIO.MessageMapper`,
`valueMapper`, `topicNameMapper`, or providing a `ConnectionFactory` must
update imports from `javax.jms.*` to
`jakarta.jms.*`. The module now requires Java 17 at runtime
([#38729](https://github.com/apache/beam/issues/38729)).
+* (Python) `Regex.find_all` with `group=Regex.ALL` now returns every group, as
`(match, group1, group2, ...)`, instead of only group 1, and with
`outputEmpty=False` it drops a match only when the whole match is empty,
matching the `group=0` behavior
([#40399](https://github.com/apache/beam/pull/40399)).
## Deprecations
diff --git a/sdks/python/apache_beam/transforms/util.py
b/sdks/python/apache_beam/transforms/util.py
index d99ebb456b5..b7885544dbf 100644
--- a/sdks/python/apache_beam/transforms/util.py
+++ b/sdks/python/apache_beam/transforms/util.py
@@ -2219,28 +2219,29 @@ class Regex(object):
@staticmethod
@typehints.with_input_types(str)
- @typehints.with_output_types(Union[list[str], list[tuple[str, str]]])
+ @typehints.with_output_types(Union[list[str], list[tuple[str, ...]]])
@ptransform_fn
def find_all(pcoll, regex, group=0, outputEmpty=True):
"""
Returns the matches if a portion of the line matches the Regex. By default,
list of group 0 will return with empty items. To get all groups, pass the
`Regex.ALL` flag in the `group` parameter which returns all the groups in
- the tuple format.
+ the tuple format. When using `Regex.ALL`, each item is formatted as a
+ tuple: `(match, group1, group2, ...)`.
Args:
regex: the regular expression string or (re.compile) pattern.
group: (optional) name of the group, it can be integer or a string value.
- outputEmpty: (optional) Should empty be output. True to output empties
- and false if not.
+ outputEmpty: (optional) Whether to allow matches with empty groups.
+ If false, these outputs will be silently dropped.
"""
regex = Regex._regex_compile(regex)
def _process(element):
matches = regex.finditer(element)
if group == Regex.ALL:
- yield [(m.group(), m.groups()[0]) for m in matches
- if outputEmpty or m.groups()[0]]
+ yield [(m.group(), *m.groups()) for m in matches
+ if outputEmpty or m.group()]
else:
yield [m.group(group) for m in matches if outputEmpty or
m.group(group)]
diff --git a/sdks/python/apache_beam/transforms/util_test.py
b/sdks/python/apache_beam/transforms/util_test.py
index e8e7d762b81..bba09245a0f 100644
--- a/sdks/python/apache_beam/transforms/util_test.py
+++ b/sdks/python/apache_beam/transforms/util_test.py
@@ -2714,9 +2714,36 @@ class RegexTest(unittest.TestCase):
pcol | 'all non empty groups' >> util.Regex.find_all(
'a(b*)', util.Regex.ALL, outputEmpty=False),
equal_to([[('ab', 'b'), ('ab', 'b'), ('ab', 'b')],
- [('abb', 'bb'), ('abbb', 'bbb')]]),
+ [('abb', 'bb'), ('a', ''), ('abbb', 'bbb')]]),
label='CheckAllNonEmptyGroups')
+ def test_find_all_groups_multiple_and_no_groups(self):
+ with TestPipeline() as p:
+ pcol = (p | beam.Create(['a=1 b=2 c=']))
+ assert_that(
+ pcol
+ | 'two groups' >> util.Regex.find_all(r'(\w)=(\d?)', util.Regex.ALL),
+ equal_to([[('a=1', 'a', '1'), ('b=2', 'b', '2'), ('c=', 'c', '')]]),
+ label='CheckTwoGroups')
+
+ assert_that(
+ pcol | 'two groups non empty match' >> util.Regex.find_all(
+ r'(\w)=(\d?)', util.Regex.ALL, outputEmpty=False),
+ equal_to([[('a=1', 'a', '1'), ('b=2', 'b', '2'), ('c=', 'c', '')]]),
+ label='CheckTwoGroupsNonEmptyMatch')
+
+ digits = (p | 'create digits' >> beam.Create(['1 2']))
+ assert_that(
+ digits | 'no groups' >> util.Regex.find_all(r'\d*', util.Regex.ALL),
+ equal_to([[('1', ), ('', ), ('2', ), ('', )]]),
+ label='CheckNoGroups')
+
+ assert_that(
+ digits | 'no groups non empty' >> util.Regex.find_all(
+ r'\d*', util.Regex.ALL, outputEmpty=False),
+ equal_to([[('1', ), ('2', )]]),
+ label='CheckNoGroupsNonEmpty')
+
def test_find_kv(self):
with TestPipeline() as p:
pcol = (p | beam.Create(['a b c d']))
