This is an automated email from the ASF dual-hosted git repository.
Abacn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 4391d8d7f38 Fix GCS glob matching for multi-byte characters (#39972)
4391d8d7f38 is described below
commit 4391d8d7f3811d7cec0b6a2c9b9f3c465265a396
Author: Shizuma5 <[email protected]>
AuthorDate: Thu Sep 3 05:17:56 2026 +0900
Fix GCS glob matching for multi-byte characters (#39972)
---
CHANGES.md | 1 +
sdks/go/pkg/beam/io/filesystem/gcs/gcs.go | 22 ++++++++++++----------
sdks/go/pkg/beam/io/filesystem/gcs/gcs_test.go | 8 ++++++++
3 files changed, 21 insertions(+), 10 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index fc34c155256..365605c7065 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -95,6 +95,7 @@
* (Java) KafkaIO dynamic reads no longer require the obsolete `beam_fn_api`
experiment ([#29998](https://github.com/apache/beam/issues/29998)).
* (Prism) Self-checkpointing splittable DoFns now resume after their requested
delay instead of immediately, so polling SDFs no longer busy-spin
([#39848](https://github.com/apache/beam/issues/39848)).
* (Java) MongoDbIO read splitting now preserves non-ObjectId `_id` types (e.g.
string ids) instead of failing to parse the generated range filters
([#39900](https://github.com/apache/beam/issues/39900)).
+* (Go) Fixed GCS glob matching silently dropping objects when the glob pattern
contains multi-byte characters
([#39969](https://github.com/apache/beam/issues/39969)).
## Security Fixes
diff --git a/sdks/go/pkg/beam/io/filesystem/gcs/gcs.go
b/sdks/go/pkg/beam/io/filesystem/gcs/gcs.go
index 8d6ce28559f..6a990f9cbe7 100644
--- a/sdks/go/pkg/beam/io/filesystem/gcs/gcs.go
+++ b/sdks/go/pkg/beam/io/filesystem/gcs/gcs.go
@@ -53,14 +53,16 @@ func globToRegex(pattern string) (*regexp.Regexp, error) {
var result strings.Builder
result.WriteString("^")
- for i := 0; i < len(pattern); i++ {
- c := pattern[i]
+ // Scan rune by rune so multi-byte characters are handled correctly.
+ runes := []rune(pattern)
+ for i := 0; i < len(runes); i++ {
+ c := runes[i]
switch c {
case '*':
// Check for ** (double asterisk)
- if i+1 < len(pattern) && pattern[i+1] == '*' {
+ if i+1 < len(runes) && runes[i+1] == '*' {
// Check if followed by / (e.g., "**/" matches
zero or more path segments)
- if i+2 < len(pattern) && pattern[i+2] == '/' {
+ if i+2 < len(runes) && runes[i+2] == '/' {
// **/ matches "" or "something/" or
"a/b/c/"
result.WriteString("(?:.*/)?")
i += 2 // Skip the second * and the /
@@ -77,26 +79,26 @@ func globToRegex(pattern string) (*regexp.Regexp, error) {
case '[':
// Character class - find the closing bracket
j := i + 1
- if j < len(pattern) && pattern[j] == '!' {
+ if j < len(runes) && runes[j] == '!' {
j++
}
- if j < len(pattern) && pattern[j] == ']' {
+ if j < len(runes) && runes[j] == ']' {
j++
}
- for j < len(pattern) && pattern[j] != ']' {
+ for j < len(runes) && runes[j] != ']' {
j++
}
- if j >= len(pattern) {
+ if j >= len(runes) {
return nil, fmt.Errorf("syntax error: unclosed
'[' in pattern %q", pattern)
} else {
// Copy the character class, converting ! to ^
for negation
result.WriteByte('[')
- content := pattern[i+1 : j]
+ content := runes[i+1 : j]
if len(content) > 0 && content[0] == '!' {
result.WriteByte('^')
content = content[1:]
}
- result.WriteString(content)
+ result.WriteString(string(content))
result.WriteByte(']')
i = j
}
diff --git a/sdks/go/pkg/beam/io/filesystem/gcs/gcs_test.go
b/sdks/go/pkg/beam/io/filesystem/gcs/gcs_test.go
index cd6aab2a236..351b0b58c13 100644
--- a/sdks/go/pkg/beam/io/filesystem/gcs/gcs_test.go
+++ b/sdks/go/pkg/beam/io/filesystem/gcs/gcs_test.go
@@ -322,6 +322,14 @@ func TestGlobToRegex(t *testing.T) {
{"file.txt", "file.txt", true},
{"file.txt", "fileXtxt", false},
{"file(1).txt", "file(1).txt", true},
+
+ // Multi-byte characters must be preserved
+ {"résumé.txt", "résumé.txt", true},
+ {"résumé.txt", "resume.txt", false},
+ {"résumé/*.txt", "résumé/file.txt", true},
+ {"*/résumé.txt", "dir/résumé.txt", true},
+ {"*.txt", "résumé.txt", true},
+ {"ïé.txt", "ïé.txt", true},
}
for _, tt := range tests {
