svn commit: r70621 [1/2] - /dev/beam/2.58.0/python/

2024-07-29 Thread jrmccluskey
Author: jrmccluskey
Date: Tue Jul 30 02:25:02 2024
New Revision: 70621

Log:
Staging Python artifacts for Apache Beam 2.58.0 RC

Modified:
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-macosx_10_9_x86_64.whl

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-macosx_10_9_x86_64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-macosx_10_9_x86_64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.sha512
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win32.whl
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win32.whl.asc
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win32.whl.sha512
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win_amd64.whl
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win_amd64.whl.asc
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win_amd64.whl.sha512
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-macosx_10_9_x86_64.whl

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-macosx_10_9_x86_64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-macosx_10_9_x86_64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.sha512
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win32.whl
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win32.whl.asc
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win32.whl.sha512
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win_amd64.whl
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win_amd64.whl.asc
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win_amd64.whl.sha512
dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-macosx_10_9_x86_64.whl

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-macosx_10_9_x86_64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-macosx_10_9_x86_64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.sha512
dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-win32.whl
dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-win32.whl.asc
dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-win32.whl.sha512
dev/beam/2.58.0/python/apache_beam-2.58.0

svn commit: r70621 [2/2] - /dev/beam/2.58.0/python/

2024-07-29 Thread jrmccluskey
Modified: 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512
==
--- 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512
 (original)
+++ 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512
 Tue Jul 30 02:25:02 2024
@@ -1 +1 @@
-36d5f59aecfd5ae9726f76718b3a2e192a09b15542a2e8c878bc437ded0f620ad91ee53a03eaa83561c7b62f953b5ea10e25d4df0224f1347181b552ed419384
  apache_beam-2.58.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
+8f702febb1f103e21f2a811786092571c86435cf3b2a14b4e4ba40b29da8baea6a7a40ff84294fd38489f6a090c566ab1953385abb9b352a3857e51dfbb780a0
  apache_beam-2.58.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl

Modified: 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl
==
Binary files - no diff available.

Modified: 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl.asc
==
--- 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl.asc
 (original)
+++ 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl.asc
 Tue Jul 30 02:25:02 2024
@@ -1,17 +1,17 @@
 -BEGIN PGP SIGNATURE-
 
-iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaYHgAYHHByaXZhdGVA
-YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiXMoP+gPtVF2oaXOyaUIp6I0JH9c5
-QJdNL9hy4cF/lqamlgSZMaG9++xDArghkoNEL79MBISQF+5A95dNlzzKWw+3EP0L
-98GoeooPxfXInXncG119HI25cff+TzBEnaZ+ql5lVnyZgr9fx6z68XaUUcbcnljJ
-asc9uFCHm6AsG2NEpCo8ZfQQjgE9y6Xx7dTtCKgff6CZXdgQodFNWHGWXkgG1AC6
-pxwKTfauU9AMelKxIc4S5gFEPXFulO3jJWC5SwAQeBd70XYHhEOevZ0RElb7WoD8
-bnxYwmSHMbJ0b/KYyRHQgMRm5wye9ZqMo9O5aqnEstNQxu8+1pFXqT6a34CMgmqk
-mKE5Cfch4DUCGKZKmnOFsIPDOcTVOIDXP4i9HxgZhPW2TVnI9QQHLskFGlVtUFrj
-OFPo5ZeTcUlHXhXDqDrJB0QtwrD2gsT+etLvKiRvB0GoTOL/DeI6PUDLmyCCUDfG
-cGJdWarXUbHPPlWxf/Ae1ZQqUjUZWjRvFhqZbzkAYgeDcLdgx1jSejFMdttkxzVS
-cRQ6ilQQ1QbAVmYvopBJGCuXs5w7nxK9SJinVMpPvv/GIwIhnKsA/iVvFMK8yKZ5
-tBeXEW7G+OlwRtJc3ZtO7QamrxzWFwuLu9hSyvkdxkRlGRH3LlKVZUPcVHwzW98p
-V/5grIZ1E5r1dcCdt86R
-=vBce
+iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaoTuoYHHByaXZhdGVA
+YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQicbIQAInIlKXSPvWIUY6d9F8ThH4w
+XqGYCwlOPsHNgYsDUb4+ho1TEQI4xJN26DZ837j4L7vvR0s+aS+8e4DLZYpYJHek
+cCzNL7TcC3E1d6l8rx59/LSM0qGGsKQmrBaQfFbmni1qA4Ifyp9JBeIqu396pS24
+IBNPnkAR3J2snBZTJTn/AcWf1Ax3qs7ToXi2WZZhMLwHcw2Ye0HxnW4vA7BpqXFN
+YJBM39VKNBnhfkfLiSQZUeQk0h3JyPda9tPfV1Y7D7FaG3G3clmwwjNfRUk8qD9p
+a3uvky5K54Q11dyFtfuca+zH4ROK+5JopGWQdzO8gVJkrvaDoNA7be7TsITDwaQL
+g8/l7Nh2uOIotCqzC1biuou5tZQjoRg429/NJA5Ce4bRpbr7xPZsipC/fMuE7PDh
+PcI7WnNtvdEmQnxz9PtZ2KVtGF+BqvqVQvUXLc0UeM67oDyQ77dsg5YHhKd6rDf+
+1hM9LqJjuQ/xZSfBbJfZNZAQjf/g9ygeoVDAIXbe0ZqBM0LjwMHdxqPatcS3CD+J
+l+mh1Ys8+AQ3BBXtDE8Zn/gHDpvJG09IiNWwbdm3gf4OLz68DI8APZx5dPFl8pQo
+VyQ9KglbzRgrM6wqfGebMajR9ViTMy//vF5/41K2IF03dUWUwVWbwhImpO4GjliS
+nsLHiGg/Iy3woC8VTRGz
+=PsTY
 -END PGP SIGNATURE-

Modified: 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl.sha512
==
--- 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl.sha512
 (original)
+++ 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl.sha512
 Tue Jul 30 02:25:02 2024
@@ -1 +1 @@
-d0af1d34061a25aace0efc544daf78570d537c164cde46349ca22ba6a61358e996c01e70f7795159d7ecfd348de56fd78c387e665bed6a05e79e88daf705923c
  apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl
+4d12ec7f155dc7144721ae383704a933a8c951cb634b2f72d2738c4177c1a4af21989b3bd805bcebf1624dc6b41aab458f879ae2224eba8624cc7c6bed234ae1
  apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl

Modified: 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
==
Binary files - no diff available.

Modified: 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc
==
--- 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc
 (original)
+++ 
dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc
 Tue Jul 30 02:25:02 2024
@@ -1,17 +1,17 @@
 -BEGIN PGP SIGNATURE-
 
-iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaYHgAYHHByaXZhdGVA
-YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQivLMP/0eN2lbK3rdNbBP2thoulGEg

svn commit: r70619 - in /dev/beam/2.58.0: apache-beam-2.58.0-source-release.zip apache-beam-2.58.0-source-release.zip.asc apache-beam-2.58.0-source-release.zip.sha512

2024-07-29 Thread jrmccluskey
Author: jrmccluskey
Date: Mon Jul 29 21:53:35 2024
New Revision: 70619

Log:
Staging Java artifacts for Apache Beam 2.58.0 RC2

Added:
dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip   (with props)
Modified:
dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc
dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512

Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip
==
Binary file - no diff available.

Propchange: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip
--
svn:mime-type = application/octet-stream

Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc
==
--- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc (original)
+++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc Mon Jul 29 
21:53:35 2024
@@ -1,17 +1,17 @@
 -BEGIN PGP SIGNATURE-
 
-iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaibOMYHHByaXZhdGVA
-YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiSh4QAIYodZ8Jh0NdXBclENiPut8p
-srQmrZoRKvBOYIBZTl9NzKNVasLRccb3jFRHPnqwSK7GMDKOoijocLpb4ZzxZk4W
-vasQfugu33l28aN7Vpx5qcHJrsoNU8md2pgcI7XiPaKocHn/sQsZtzfrfLZUA6iA
-/tcZV73kfjKFem/UEeYJbKFfg2GOQjUM8HpUnlV5Jr02FZt2jSnsFApbPZ3RxgSj
-5ORE/nH1EiatM71NcHRiRlfY53rub+v6cNAiqgz5SGcFGRX6LbWx7GrnIaMMgtKN
-rnaiSfk7ZdjJjiogRIuavdPzvEwCv83odgJd4wi+f+HNPteqJJoisGvnyObj5fs9
-FfI4LXWBcSq2bxjIV5ueKZIQSCbXQIm366E/7P8940V47NSAYRW6NPqsTaY1RQ7c
-Ij/Jcbm7oCQQ0xIcQ34RMbAWIee5B9aedGZR60bqqeuzk37e3bkW9yU8uvPUJxW/
-5mx1ockcZNGCecZXzN9IP+l9swOkb23c9k8sUKV1p9dXqBCxTRyZe6z88iq0+Ifd
-kBRJ74pMv4Xp/dHemtBx3oAybcBhkdV8k8IdM/OCxIrYXtT9zpuvoS2AoY1aJoVD
-aAmiE5N6etD8aRd7DT1seclZPTmeVXHnus4cacMwQhAalTu5OSn4pmcNFVg3Aed3
-M4XbjSl3AX243AVykCPX
-=olUN
+iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaoD1wYHHByaXZhdGVA
+YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQixd4P/1jBZ3Sf5zGvvkXZbvkWVZyk
+Nw0pIkjXXCIrb6JoZYyIisCHbqF9fpTxF0IDCw2JhMT/0GewdYIFkY9jO4SWtN4j
+qAg+rUcOnEHBjOH5Kekt+EoI5CfJzqqxkO9RNsqmEigLSeIf+le8aEVKaswGcgtq
+wCjwJv147HajM48/w0OnknP36oUoA6yVO0BLOZl0rF/dX/ucBm1LhRoWymKBAOim
+SeuOaf3WIpPEqV0O8wsKT+OvmfoEpRh5KApAgHWyMINdFE1bCjxZxkCArJ8rPmSS
+7sKlmHiJ9ehZmXW/tdDygI4sP0s0I1jqdjFwUB01w91wjBaxe0EzzVwihRlNOggT
+8Gvt78svzwkvHNXKxyIY27coHFLggimNitxEUXKk8tCjIOulw3nJKEe/WAptnqau
+/o0sxcLRfyczEIcH6AG/wa29sCrYvg4i4nL9aPItqQaqC3v52GrML7G17zPqoejk
+VEapnyAJXAYZdGS2O5GUy3u+JTLDFBE4SFTntCQP5IQM9/keFAa3x/wRbjcKVMco
+P2/CiNpwh2jE7RrWirz6DKpT7qh/p2aByIymOcDywit0vecR5tCcvzWvAvt3WU24
+QPAh5OzovQW9Q6O2Xriye+XTld+Ig038nqaj6K7L2UIi9sxoPWAMPPeg2kEH3kPz
+CsHbHsCx7rn4Ws5fxttb
+=mW3A
 -END PGP SIGNATURE-

Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512
==
--- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 (original)
+++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 Mon Jul 29 
21:53:35 2024
@@ -1 +1 @@
-2a7ecfe76bebf27555b9357a8b6cc22e5b6fe0e68229028b9e3a8cfc3480e3609d19c04c686b601630e9b08c6a0094af60313076588d454e79dd9e907045c085
  apache-beam-2.58.0-source-release.zip
+01a37fe442e3f75617aee2b9e91006849e5e497e2fdddc3b14ed4874b1662018dffa9caa548f04458fc23b86ee04219e975ab579e5d0e9098ba1045d97c680e7
  apache-beam-2.58.0-source-release.zip




(beam) annotated tag v2.58.0-RC2 updated (d8315f63270 -> 5867feee6a5)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag v2.58.0-RC2
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag v2.58.0-RC2 was modified! ***

from d8315f63270 (commit)
  to 5867feee6a5 (tag)
 tagging d8315f63270b6b4d79afd2579b41dba23b999235 (commit)
 replaces jupyterlab-sidepanel-v3.0.0
  by Jack McCluskey
  on Mon Jul 29 17:45:20 2024 -0400

- Log -
v2.58.0-RC2
---


No new revisions were added by this update.

Summary of changes:



(beam) annotated tag sdks/v2.58.0-RC2 updated (d8315f63270 -> f1090e66603)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC2
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag sdks/v2.58.0-RC2 was modified! ***

from d8315f63270 (commit)
  to f1090e66603 (tag)
 tagging d8315f63270b6b4d79afd2579b41dba23b999235 (commit)
 replaces jupyterlab-sidepanel-v3.0.0
  by Jack McCluskey
  on Mon Jul 29 17:45:20 2024 -0400

- Log -
Go SDK v2.58.0-RC2
---


No new revisions were added by this update.

Summary of changes:



(beam) annotated tag sdks/v2.58.0-RC2 deleted (was 5b9a1016dd3)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC2
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag sdks/v2.58.0-RC2 was deleted! ***

   tag was  5b9a1016dd3

This change permanently discards the following revisions:

 discard 5782d1fd34f Set version for 2.58.0 RC2



(beam) annotated tag v2.58.0-RC2 deleted (was bbfbcc7d1f3)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag v2.58.0-RC2
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag v2.58.0-RC2 was deleted! ***

   tag was  bbfbcc7d1f3

This change permanently discards the following revisions:

 discard 5782d1fd34f Set version for 2.58.0 RC2



(beam) branch release-2.58.0 updated: Adds ordered list user state support to fnapi accessor cache. (#31923) (#32013)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/release-2.58.0 by this push:
 new 7fbbe0bf26a Adds ordered list user state support to fnapi accessor 
cache. (#31923) (#32013)
7fbbe0bf26a is described below

commit 7fbbe0bf26ae6bf1112efcd0ec5814b03c7b9f0a
Author: Shunping Huang 
AuthorDate: Mon Jul 29 17:43:25 2024 -0400

Adds ordered list user state support to fnapi accessor cache. (#31923) 
(#32013)

Co-authored-by: Andrew Crites 
---
 .../java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java | 8 +---
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git 
a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java
 
b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java
index 93f89301d15..5b304890b35 100644
--- 
a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java
+++ 
b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java
@@ -813,14 +813,8 @@ public class FnApiStateAccessor implements 
SideInputReader, StateBinder {
   private Cache getCacheFor(StateKey stateKey) {
 switch (stateKey.getTypeCase()) {
   case BAG_USER_STATE:
-for (CacheToken token : cacheTokens.get()) {
-  if (!token.hasUserState()) {
-continue;
-  }
-  return Caches.subCache(processWideCache, token, stateKey);
-}
-break;
   case MULTIMAP_KEYS_USER_STATE:
+  case ORDERED_LIST_USER_STATE:
 for (CacheToken token : cacheTokens.get()) {
   if (!token.hasUserState()) {
 continue;



(beam) branch master updated (9e431b49bc7 -> 5d13894975b)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


from 9e431b49bc7 Correctly chose earliest or latest in pane. (#31979)
 add 5d13894975b Revert "Update Build Wheels to only build once on RCs" 
(#32014)

No new revisions were added by this update.

Summary of changes:
 .github/workflows/build_wheels.yml | 5 +
 1 file changed, 1 insertion(+), 4 deletions(-)



(beam) branch revert-32010-wheelCherrypick updated (5afeaf4e13e -> 65022e988e2)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch revert-32010-wheelCherrypick
in repository https://gitbox.apache.org/repos/asf/beam.git


from 5afeaf4e13e Revert "Update Build Wheels to only build once on RCs 
(#32009) (#32010)"
 add 65022e988e2 Add 8 hour timeout

No new revisions were added by this update.

Summary of changes:
 .github/workflows/build_wheels.yml | 1 +
 1 file changed, 1 insertion(+)



(beam) branch revert-32009-bigWheels updated (9aabe2e07c1 -> 54971749d83)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch revert-32009-bigWheels
in repository https://gitbox.apache.org/repos/asf/beam.git


from 9aabe2e07c1 Revert "Update Build Wheels to only build once on RCs 
(#32009)"
 add 54971749d83 Add 8 hour timeout

No new revisions were added by this update.

Summary of changes:
 .github/workflows/build_wheels.yml | 1 +
 1 file changed, 1 insertion(+)



(beam) 01/01: Revert "Update Build Wheels to only build once on RCs (#32009) (#32010)"

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch revert-32010-wheelCherrypick
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 5afeaf4e13e656ae0183969a691c73576c9cf7cf
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Mon Jul 29 16:50:28 2024 -0400

Revert "Update Build Wheels to only build once on RCs (#32009) (#32010)"

This reverts commit 0a4bdc370ae6532151bedf296286be34d1c13473.
---
 .github/workflows/build_wheels.yml | 4 
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/build_wheels.yml 
b/.github/workflows/build_wheels.yml
index 35a21626d2a..dc14359aa88 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -236,7 +236,6 @@ jobs:
 arch: aarch64
 steps:
 - name: Download python source distribution from artifacts
-  if: ${{ needs.build_source.outputs.is_rc == 0 }}
   # Pinned to v3 because of 
https://github.com/actions/download-artifact/issues/249
   uses: actions/download-artifact@v3
   with:
@@ -260,7 +259,6 @@ jobs:
   # note: sync cibuildwheel version with gradle task sdks:python:bdistPy* 
steps
   run: pip install cibuildwheel==2.17.0 setuptools
 - name: Build wheel
-  if: ${{ needs.build_source.outputs.is_rc == 0 }}
   working-directory: apache-beam-source
   env:
 CIBW_BUILD: ${{ matrix.os_python.python }}
@@ -273,7 +271,6 @@ jobs:
   if: startsWith(matrix.os_python.os, 'macos')
   run: brew install coreutils
 - name: Add checksums
-  if: ${{ needs.build_source.outputs.is_rc == 0 }}
   working-directory: apache-beam-source/wheelhouse/
   run: |
 for file in *.whl; do
@@ -281,7 +278,6 @@ jobs:
 done
   shell: bash
 - name: Upload wheels as artifacts
-  if: ${{ needs.build_source.outputs.is_rc == 0 }}
   # Pinned to v3 because of 
https://github.com/actions/download-artifact/issues/249
   uses: actions/upload-artifact@v3
   with:



(beam) branch revert-32010-wheelCherrypick created (now 5afeaf4e13e)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch revert-32010-wheelCherrypick
in repository https://gitbox.apache.org/repos/asf/beam.git


  at 5afeaf4e13e Revert "Update Build Wheels to only build once on RCs 
(#32009) (#32010)"

This branch includes the following new commits:

 new 5afeaf4e13e Revert "Update Build Wheels to only build once on RCs 
(#32009) (#32010)"

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.




(beam) branch revert-32009-bigWheels created (now 9aabe2e07c1)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch revert-32009-bigWheels
in repository https://gitbox.apache.org/repos/asf/beam.git


  at 9aabe2e07c1 Revert "Update Build Wheels to only build once on RCs 
(#32009)"

This branch includes the following new commits:

 new 9aabe2e07c1 Revert "Update Build Wheels to only build once on RCs 
(#32009)"

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.




(beam) 01/01: Revert "Update Build Wheels to only build once on RCs (#32009)"

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch revert-32009-bigWheels
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 9aabe2e07c1b562c1edd4a5c1074056d4c924d8f
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Mon Jul 29 16:49:42 2024 -0400

Revert "Update Build Wheels to only build once on RCs (#32009)"

This reverts commit ffae5b5bf0b988a9db6e5a1ea93d6ba75e438d08.
---
 .github/workflows/build_wheels.yml | 4 
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/build_wheels.yml 
b/.github/workflows/build_wheels.yml
index 35a21626d2a..dc14359aa88 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -236,7 +236,6 @@ jobs:
 arch: aarch64
 steps:
 - name: Download python source distribution from artifacts
-  if: ${{ needs.build_source.outputs.is_rc == 0 }}
   # Pinned to v3 because of 
https://github.com/actions/download-artifact/issues/249
   uses: actions/download-artifact@v3
   with:
@@ -260,7 +259,6 @@ jobs:
   # note: sync cibuildwheel version with gradle task sdks:python:bdistPy* 
steps
   run: pip install cibuildwheel==2.17.0 setuptools
 - name: Build wheel
-  if: ${{ needs.build_source.outputs.is_rc == 0 }}
   working-directory: apache-beam-source
   env:
 CIBW_BUILD: ${{ matrix.os_python.python }}
@@ -273,7 +271,6 @@ jobs:
   if: startsWith(matrix.os_python.os, 'macos')
   run: brew install coreutils
 - name: Add checksums
-  if: ${{ needs.build_source.outputs.is_rc == 0 }}
   working-directory: apache-beam-source/wheelhouse/
   run: |
 for file in *.whl; do
@@ -281,7 +278,6 @@ jobs:
 done
   shell: bash
 - name: Upload wheels as artifacts
-  if: ${{ needs.build_source.outputs.is_rc == 0 }}
   # Pinned to v3 because of 
https://github.com/actions/download-artifact/issues/249
   uses: actions/upload-artifact@v3
   with:



(beam) branch master updated: Regenerate Dataflow Python client (#31997)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 0a65b4b065b Regenerate Dataflow Python client (#31997)
0a65b4b065b is described below

commit 0a65b4b065b27163d6839f9924844c139c7fdf68
Author: Yi Hu 
AuthorDate: Mon Jul 29 13:08:18 2024 -0400

Regenerate Dataflow Python client (#31997)
---
 .../clients/dataflow/dataflow_v1b3_client.py   |   81 +-
 .../clients/dataflow/dataflow_v1b3_messages.py | 1497 
 2 files changed, 1224 insertions(+), 354 deletions(-)

diff --git 
a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
 
b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
index cc982098797..e42b180bbec 100644
--- 
a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
+++ 
b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
@@ -1,8 +1,5 @@
 """Generated client library for dataflow version v1b3."""
 # NOTE: This file is autogenerated and should not be edited by hand.
-
-from __future__ import absolute_import
-
 from apitools.base.py import base_api
 from . import dataflow_v1b3_messages as messages
 
@@ -17,9 +14,7 @@ class DataflowV1b3(base_api.BaseApiClient):
   _PACKAGE = 'dataflow'
   _SCOPES = [
   'https://www.googleapis.com/auth/cloud-platform',
-  'https://www.googleapis.com/auth/compute',
-  'https://www.googleapis.com/auth/compute.readonly',
-  'https://www.googleapis.com/auth/userinfo.email'
+  'https://www.googleapis.com/auth/compute'
   ]
   _VERSION = 'v1b3'
   _CLIENT_ID = '1042881264118.apps.googleusercontent.com'
@@ -75,7 +70,6 @@ class DataflowV1b3(base_api.BaseApiClient):
 self.projects_locations_jobs = self.ProjectsLocationsJobsService(self)
 self.projects_locations_snapshots = self.ProjectsLocationsSnapshotsService(
 self)
-self.projects_locations_sql = self.ProjectsLocationsSqlService(self)
 self.projects_locations_templates = self.ProjectsLocationsTemplatesService(
 self)
 self.projects_locations = self.ProjectsLocationsService(self)
@@ -254,7 +248,7 @@ class DataflowV1b3(base_api.BaseApiClient):
   self._upload_configs = {}
 
 def Aggregated(self, request, global_params=None):
-  r"""List the jobs of a project across all regions.
+  r"""List the jobs of a project across all regions. **Note:** This method 
doesn't support filtering the list of jobs by name.
 
   Args:
 request: (DataflowProjectsJobsAggregatedRequest) input message
@@ -270,7 +264,8 @@ class DataflowV1b3(base_api.BaseApiClient):
 method_id='dataflow.projects.jobs.aggregated',
 ordered_params=['projectId'],
 path_params=['projectId'],
-query_params=['filter', 'location', 'pageSize', 'pageToken', 'view'],
+query_params=
+['filter', 'location', 'name', 'pageSize', 'pageToken', 'view'],
 relative_path='v1b3/projects/{projectId}/jobs:aggregated',
 request_field='',
 request_type_name='DataflowProjectsJobsAggregatedRequest',
@@ -279,7 +274,7 @@ class DataflowV1b3(base_api.BaseApiClient):
 )
 
 def Create(self, request, global_params=None):
-  r"""Creates a Cloud Dataflow job. To create a job, we recommend using 
`projects.locations.jobs.create` with a [regional endpoint] 
(https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). Using 
`projects.jobs.create` is not recommended, as your job will always start in 
`us-central1`.
+  r"""Creates a Cloud Dataflow job. To create a job, we recommend using 
`projects.locations.jobs.create` with a [regional endpoint] 
(https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). Using 
`projects.jobs.create` is not recommended, as your job will always start in 
`us-central1`. Do not enter confidential information when you supply string 
values using the API.
 
   Args:
 request: (DataflowProjectsJobsCreateRequest) input message
@@ -354,7 +349,7 @@ class DataflowV1b3(base_api.BaseApiClient):
 )
 
 def List(self, request, global_params=None):
-  r"""List the jobs of a project. To list the jobs of a project in a 
region, we recommend using `projects.locations.jobs.list` with a [regional 
endpoint] (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). 
To list the all jobs across all regions, use `projects.jobs.aggregated`. Using 
`projects.jobs.list` is not recommended, as you can only get the list of jobs 
that are running in `us-central1`.
+  r"""List the jobs of a project. To list the jobs of a project in a 
region, we recommend using `proje

(beam) annotated tag sdks/v2.58.0-RC2 updated (5782d1fd34f -> 5b9a1016dd3)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC2
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag sdks/v2.58.0-RC2 was modified! ***

from 5782d1fd34f (commit)
  to 5b9a1016dd3 (tag)
 tagging 5782d1fd34f53027ccc155a2ca70d27086b629cf (commit)
 replaces jupyterlab-sidepanel-v3.0.0
  by Jack McCluskey
  on Mon Jul 29 10:29:27 2024 -0400

- Log -
Go SDK v2.58.0-RC2
---


No new revisions were added by this update.

Summary of changes:



(beam) annotated tag v2.58.0-RC2 updated (5782d1fd34f -> bbfbcc7d1f3)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag v2.58.0-RC2
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag v2.58.0-RC2 was modified! ***

from 5782d1fd34f (commit)
  to bbfbcc7d1f3 (tag)
 tagging 5782d1fd34f53027ccc155a2ca70d27086b629cf (commit)
 replaces jupyterlab-sidepanel-v3.0.0
  by Jack McCluskey
  on Mon Jul 29 10:29:27 2024 -0400

- Log -
v2.58.0-RC2
---


No new revisions were added by this update.

Summary of changes:



(beam) annotated tag sdks/v2.58.0-RC2 deleted (was 6e4316aa3d3)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC2
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag sdks/v2.58.0-RC2 was deleted! ***

   tag was  6e4316aa3d3

This change permanently discards the following revisions:

 discard 898f723b951 Set version for 2.58.0 RC2



(beam) annotated tag v2.58.0-RC2 deleted (was 5bf106b32d7)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag v2.58.0-RC2
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag v2.58.0-RC2 was deleted! ***

   tag was  5bf106b32d7

The revisions that were on this annotated tag are still contained in
other references; therefore, this change does not discard any commits
from the repository.



(beam) branch master updated (7930a1fc88b -> ffae5b5bf0b)

2024-07-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


from 7930a1fc88b Revert "Avoid publishing string set metrics on the 
Dataflow legacy runner." (#32002)
 add ffae5b5bf0b Update Build Wheels to only build once on RCs (#32009)

No new revisions were added by this update.

Summary of changes:
 .github/workflows/build_wheels.yml | 4 
 1 file changed, 4 insertions(+)



(beam-site) branch updates_release_2.58.0 updated (c9292ee7bc -> 055335f97f)

2024-07-25 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch updates_release_2.58.0
in repository https://gitbox.apache.org/repos/asf/beam-site.git


omit c9292ee7bc Update beam-site for release 2.58.0.
 add 055335f97f Update beam-site for release 2.58.0.

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (c9292ee7bc)
\
 N -- N -- N   refs/heads/updates_release_2.58.0 (055335f97f)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

No new revisions were added by this update.

Summary of changes:
 javadoc/2.58.0/deprecated-list.html|  28 +-
 .../beam/sdk/expansion/service/package-tree.html   |   2 +-
 .../sql/impl/rel/BeamLogicalConvention.html|   2 +-
 .../sdk/extensions/sql/impl/rel/package-tree.html  |   2 +-
 .../org/apache/beam/sdk/io/cdap/package-tree.html  |   2 +-
 .../org/apache/beam/sdk/io/fs/package-tree.html|   4 +-
 .../beam/sdk/io/gcp/bigquery/package-tree.html |  10 +-
 .../beam/sdk/io/gcp/healthcare/package-tree.html   |   4 +-
 .../spanner/changestreams/model/package-tree.html  |   2 +-
 .../org/apache/beam/sdk/io/package-tree.html   |   6 +-
 .../beam/sdk/io/snowflake/enums/package-tree.html  |   2 +-
 .../org/apache/beam/sdk/options/package-tree.html  |  24 +-
 .../2.58.0/org/apache/beam/sdk/package-tree.html   |   2 +-
 .../beam/sdk/schemas/annotations/package-tree.html |   6 +-
 .../org/apache/beam/sdk/schemas/package-tree.html  |   2 +-
 .../org/apache/beam/sdk/transforms/DoFn.html   |   2 +-
 .../sdk/transforms/display/HasDisplayData.html |   2 +-
 .../apache/beam/sdk/transforms/package-tree.html   |  48 ++--
 .../sdk/transforms/windowing/package-tree.html |   4 +-
 .../org/apache/beam/sdk/values/package-tree.html   |   2 +-
 javadoc/2.58.0/overview-tree.html  | 106 
 .../_modules/apache_beam/yaml/yaml_transform.html  |   4 +-
 pydoc/2.58.0/apache_beam.dataframe.frames.html | 294 ++---
 pydoc/2.58.0/apache_beam.dataframe.io.html |  10 +-
 pydoc/2.58.0/apache_beam.dataframe.transforms.html |  12 +-
 pydoc/2.58.0/apache_beam.io.fileio.html|   2 +-
 ...he_beam.ml.inference.huggingface_inference.html |  16 +-
 .../apache_beam.ml.inference.onnx_inference.html   |   6 +-
 ...apache_beam.ml.inference.pytorch_inference.html |  16 +-
 ...che_beam.ml.inference.tensorflow_inference.html |  14 +-
 ...pache_beam.ml.inference.tensorrt_inference.html |   6 +-
 ...apache_beam.ml.inference.xgboost_inference.html |  22 +-
 .../2.58.0/apache_beam.ml.transforms.handlers.html |   8 +-
 pydoc/2.58.0/apache_beam.ml.transforms.tft.html|  26 +-
 .../apache_beam.transforms.periodicsequence.html   |   2 +-
 pydoc/2.58.0/apache_beam.typehints.schemas.html|   4 +-
 pydoc/2.58.0/doctest/output.txt|  28 +-
 .../2.58.0/doctrees/apache_beam.io.fileio.doctree  | Bin 83008 -> 83008 bytes
 ...beam.ml.inference.huggingface_inference.doctree | Bin 124656 -> 124656 bytes
 ...apache_beam.ml.inference.onnx_inference.doctree | Bin 31312 -> 31312 bytes
 ...che_beam.ml.inference.pytorch_inference.doctree | Bin 91725 -> 91725 bytes
 ..._beam.ml.inference.tensorflow_inference.doctree | Bin 76300 -> 76300 bytes
 ...he_beam.ml.inference.tensorrt_inference.doctree | Bin 43210 -> 43210 bytes
 ...che_beam.ml.inference.xgboost_inference.doctree | Bin 118378 -> 118378 bytes
 .../apache_beam.ml.transforms.handlers.doctree | Bin 29035 -> 29035 bytes
 .../doctrees/apache_beam.ml.transforms.tft.doctree | Bin 128840 -> 128840 bytes
 ...apache_beam.transforms.periodicsequence.doctree | Bin 37699 -> 37699 bytes
 .../doctrees/apache_beam.typehints.schemas.doctree | Bin 223622 -> 223622 bytes
 pydoc/2.58.0/doctrees/environment.pickle   | Bin 13109915 -> 13110178 
bytes
 pydoc/2.58.0/searchindex.js|   2 +-
 .../classes/coders_js_coders.BsonObjectCoder.html  |  10 +-
 .../coders_js_coders.GeneralObjectCoder.html   |  16 +-
 .../classes/coders_required_coders.BytesCoder.html |  14 +-
 ...ers_required_coders.FullWindowedValueCoder.html |  18 +-
 .../coders_required_coders.GlobalWindowCoder.html  |  12 +-
 .../coders_required_coders.InstantCoder.html   |  12 +-
 .../coders_required_coders.IterableCoder.html  |  18 +-
 .../classes/coders_required_coders.KVCoder.html|  18 +-
 ...coders_required_coders.LengthPrefixedCoder.html

svn commit: r70549 - in /dev/beam/2.58.0/prism: darwin/amd64/ darwin/arm64/ linux/amd64/ linux/arm64/ windows/amd64/ windows/arm64/

2024-07-25 Thread jrmccluskey
Author: jrmccluskey
Date: Thu Jul 25 15:22:08 2024
New Revision: 70549

Log:
Staging Prism artifacts for Apache Beam 2.58.0 RC

Modified:

dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip

dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc

dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512

dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip

dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip.asc

dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip.sha512
dev/beam/2.58.0/prism/linux/amd64/apache_beam-v2.58.0-prism-linux-amd64.zip

dev/beam/2.58.0/prism/linux/amd64/apache_beam-v2.58.0-prism-linux-amd64.zip.asc

dev/beam/2.58.0/prism/linux/amd64/apache_beam-v2.58.0-prism-linux-amd64.zip.sha512
dev/beam/2.58.0/prism/linux/arm64/apache_beam-v2.58.0-prism-linux-arm64.zip

dev/beam/2.58.0/prism/linux/arm64/apache_beam-v2.58.0-prism-linux-arm64.zip.asc

dev/beam/2.58.0/prism/linux/arm64/apache_beam-v2.58.0-prism-linux-arm64.zip.sha512

dev/beam/2.58.0/prism/windows/amd64/apache_beam-v2.58.0-prism-windows-amd64.zip

dev/beam/2.58.0/prism/windows/amd64/apache_beam-v2.58.0-prism-windows-amd64.zip.asc

dev/beam/2.58.0/prism/windows/amd64/apache_beam-v2.58.0-prism-windows-amd64.zip.sha512

dev/beam/2.58.0/prism/windows/arm64/apache_beam-v2.58.0-prism-windows-arm64.zip

dev/beam/2.58.0/prism/windows/arm64/apache_beam-v2.58.0-prism-windows-arm64.zip.asc

dev/beam/2.58.0/prism/windows/arm64/apache_beam-v2.58.0-prism-windows-arm64.zip.sha512

Modified: 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip
==
Binary files - no diff available.

Modified: 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc
==
--- 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc
 (original)
+++ 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc
 Thu Jul 25 15:22:08 2024
@@ -1,17 +1,17 @@
 -BEGIN PGP SIGNATURE-
 
-iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaW4MMYHHByaXZhdGVA
-YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiy5oP/RmV0UdqiUOSuoHy7WhrJEle
-xfW0HIiXguSAUCxMmVOoVbtuHk8lmBcpXkQqbDvTqoaPZbLqFST3nFpf8hxeAmCW
-mjiNyZ5fkYjjLVPR7BftjGNzRBoHsjX0B28hwsgA26O2BT7CJdXtGT31xzg413hK
-WFoI3UEAkJV1LiSkePCFKmnoPIN+642GztD+LQ7KB32S/Va33i1RjNNS1Wt5mhSh
-Rv5gdOCdXBk73nm5Sfvk9PREE6Q3nJzP9ts4Nng0QVJNaXfz4mcWl4UiGbdKr/8d
-pAYaKABse5bxL+MStsjj0ZcshUIHHPN9Xs/qqguzfGSmMFFvw88fHYS4xfg/XuKk
-BMCokNFfkUrTq74j02k13CTvvayqjuyu/l42sn7W2r0M+6XoTtivHSruSvJechLc
-NfiJ4DrUctLAfk133LiCmP0j76kpk6gq5RdMlRSPq9Dz3deE0VcF0egTH/mfUMOa
-tUP1VWru8IO0wmfWX+qNM+KpZ+U28bFYpPb4Vzx2BljSgXtPk8BQmciyjw5TVEQd
-M458oh+rHU+T9zx8n+TFkGmodyx0/0khmq8LbrRsNGaPNiZs98QH9TfTe7ahYTht
-3QirvJqAKQES1GYGNlZI30dH3v1rJvgo4IJ9TkuFc3pFxUe5gViVpb73LQRQf0CX
-V23uewV1pHiXXd5vUDe+
-=Z3/h
+iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaibX0YHHByaXZhdGVA
+YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQivU0QAKbc+uOGdHH0QgRZwx/S57Ri
+YhQVaZ8Nw0FctPUAKOgYCkT4fopwXTPQU1DoCnx6nlFmnYWeHWyr1gDZWyitmuzo
+/gCx+RPfWFvsd5LEUOcfurf/JVwRv7dFeFEoSPRZssV+PsHi8bNfQmmOmZ9TGJVt
+E355S68xZM7RDVPtHNVOdkN1INMmrcw1zpuuGfIrkBxnbUwPMn/LOG3DH9kmlSzH
+IbqSvYx/hj17l/nFPqSCuy+XlM2G0gXyPA7Q1X/kZtxk5T+i8h3ZDt7CpGkXyB/B
+mFX01UwCaZe6Od4Z555TOeLvSP+gvZTJjyZKETyy/6LhRL4IuKteBjhgnPeCwr6m
+LCP0kW9NCmdJ+PujrEFnBjV8wb32dtMWdhce0GduDiilHLD2wVZJFvDGfnzbrHh0
+jrSSGTcY4QepPEiE5NNNzIwLMH4lyBBNyVzUvt/2pInkd4+gR3dIZZGc5YGtrl3V
+B3Q2qftqF9rxxxp8MTK/cHScB59PZDpn/1NiQUzKLMyiouqOmOSORpHrthvVwHn4
+CN2MSnbWPLegN6tE8gOyptYPDzwklXpK4UL7OC4awOYTebct+D3Z3IsQSTReezhF
+aD1q2zmg7QL5qrtY6v/j3km0kR5Mw8apTcQHT9LtoyGchUjUzAvIX5jvjyNybzfD
+UFrUd38o9fIRIFHE7j4y
+=aowS
 -END PGP SIGNATURE-

Modified: 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512
==
--- 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512
 (original)
+++ 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512
 Thu Jul 25 15:22:08 2024
@@ -1 +1 @@
-a76dc4a051dbbaee8e85073b56921302dce28f619d91be532d4a3b80183d666a9d53cf0f1d01df2dc962e9f7501775f9af0ac29a9104ab7b03223f239ea0605a
  apache_beam-v2.58.0-prism-darwin-amd64.zip
+8a90eb703f047964da820531318e87c9d90bdaa28e945c54ef5397b52d88e467704d5a29cf048834f266c6c2b16f5e2055ff72ffccce60e297a1f56ba35547cd
  apache_beam-v2.58.0-prism-darwin-amd64.zip

Modified: 
dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip
==
Binary files - no diff available.

Modified: 
dev/beam/2.58.0/prism

svn commit: r70548 - in /dev/beam/2.58.0: apache-beam-2.58.0-source-release.zip apache-beam-2.58.0-source-release.zip.asc apache-beam-2.58.0-source-release.zip.sha512

2024-07-25 Thread jrmccluskey
Author: jrmccluskey
Date: Thu Jul 25 15:19:03 2024
New Revision: 70548

Log:
Staging Java artifacts for Apache Beam 2.58.0 RC2

Added:
dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip   (with props)
Modified:
dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc
dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512

Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip
==
Binary file - no diff available.

Propchange: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip
--
svn:mime-type = application/octet-stream

Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc
==
--- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc (original)
+++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc Thu Jul 25 
15:19:03 2024
@@ -1,17 +1,17 @@
 -BEGIN PGP SIGNATURE-
 
-iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaW3zMYHHByaXZhdGVA
-YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQin6MP/3mX5HNCLLAheR94p5jWsobu
-UduIN/nrXKYOcDAuulm9bgOAfQgJh/4P8Ds60uf5vFCYtPGWwR3SFOf3jBFgn71c
-A990YYS2uU1vCkgI1hHASHNaViV6vuTUre2qUK5nZDRr5GH66GxIQPsIVy/mKojx
-9A7gaAueKTG4EmqF5zYJr23vk6ZKEGVBV8mjpsZ5MNsV0NmKtHuzuTX9o4JPAcVa
-8jHfyeRfLnzDhLEeGTa30NbN2ofk4YnDrHgsyBfb3DXyBZd5nGxpPTexAH3ehJG7
-XPasWTCo/024XSQwfEhkAcYSCn0t+XgJe/axq+VPC2E9Sdp2lr+MNudYlmefBUIL
-YvMM0AazTuYJajJQCBho7Zx5W1aYY1CFM73Af9N/9Fs9i5KfcPeyKvm9ehDC+CD6
-HOXzejGtPBJ3T44DWz/Im+2c6mktDAXmd6adNTw20uxGDYMhESI08cA00pxwTTGA
-NLLk0uAQ+QGB0YaLRH7svbZt41pHSTNyVlZWNrKYYGt5Kce3pWGTkyIJc0MBWpII
-pS9UmzMm1sjcogDnGcTWl78VLCQby9kgzKyB6DBAJbh63CAUkR/k8jhXqXVz6Q1p
-JChap4jSLn0GKnHtSrE1XDXcTUkd/EVu3ASr8564QdpQbhs3nKk6luwtp8DaEyg3
-HA7cTMaUYVsRu1hSdrQL
-=zDx7
+iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaibOMYHHByaXZhdGVA
+YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiSh4QAIYodZ8Jh0NdXBclENiPut8p
+srQmrZoRKvBOYIBZTl9NzKNVasLRccb3jFRHPnqwSK7GMDKOoijocLpb4ZzxZk4W
+vasQfugu33l28aN7Vpx5qcHJrsoNU8md2pgcI7XiPaKocHn/sQsZtzfrfLZUA6iA
+/tcZV73kfjKFem/UEeYJbKFfg2GOQjUM8HpUnlV5Jr02FZt2jSnsFApbPZ3RxgSj
+5ORE/nH1EiatM71NcHRiRlfY53rub+v6cNAiqgz5SGcFGRX6LbWx7GrnIaMMgtKN
+rnaiSfk7ZdjJjiogRIuavdPzvEwCv83odgJd4wi+f+HNPteqJJoisGvnyObj5fs9
+FfI4LXWBcSq2bxjIV5ueKZIQSCbXQIm366E/7P8940V47NSAYRW6NPqsTaY1RQ7c
+Ij/Jcbm7oCQQ0xIcQ34RMbAWIee5B9aedGZR60bqqeuzk37e3bkW9yU8uvPUJxW/
+5mx1ockcZNGCecZXzN9IP+l9swOkb23c9k8sUKV1p9dXqBCxTRyZe6z88iq0+Ifd
+kBRJ74pMv4Xp/dHemtBx3oAybcBhkdV8k8IdM/OCxIrYXtT9zpuvoS2AoY1aJoVD
+aAmiE5N6etD8aRd7DT1seclZPTmeVXHnus4cacMwQhAalTu5OSn4pmcNFVg3Aed3
+M4XbjSl3AX243AVykCPX
+=olUN
 -END PGP SIGNATURE-

Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512
==
--- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 (original)
+++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 Thu Jul 25 
15:19:03 2024
@@ -1 +1 @@
-48b6f6240337f5ea96c7bc9b44de93d581493ff5a4e78006e1c514810853ba972bc9325073b635b16b32c88fbdda8c4f87cb822e8e03de8ebd4cd5fc4254de4f
  apache-beam-2.58.0-source-release.zip
+2a7ecfe76bebf27555b9357a8b6cc22e5b6fe0e68229028b9e3a8cfc3480e3609d19c04c686b601630e9b08c6a0094af60313076588d454e79dd9e907045c085
  apache-beam-2.58.0-source-release.zip




(beam) annotated tag v2.58.0-RC2 updated (898f723b951 -> 5bf106b32d7)

2024-07-25 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag v2.58.0-RC2
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag v2.58.0-RC2 was modified! ***

from 898f723b951 (commit)
  to 5bf106b32d7 (tag)
 tagging 898f723b951df341a2df800a47ec6959045df49e (commit)
 replaces jupyterlab-sidepanel-v3.0.0
  by jrmccluskey
  on Thu Jul 25 15:15:31 2024 +

- Log -
v2.58.0-RC2
---


No new revisions were added by this update.

Summary of changes:



(beam) annotated tag sdks/v2.58.0-RC2 updated (898f723b951 -> 6e4316aa3d3)

2024-07-25 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC2
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag sdks/v2.58.0-RC2 was modified! ***

from 898f723b951 (commit)
  to 6e4316aa3d3 (tag)
 tagging 898f723b951df341a2df800a47ec6959045df49e (commit)
 replaces jupyterlab-sidepanel-v3.0.0
  by jrmccluskey
  on Thu Jul 25 15:15:31 2024 +

- Log -
Go SDK v2.58.0-RC2
---


No new revisions were added by this update.

Summary of changes:



(beam) branch release-2.58.0 updated: [yaml] Fix yaml provider schema validation and merging (#31974) (#31984)

2024-07-25 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/release-2.58.0 by this push:
 new cdc19d07da9 [yaml] Fix yaml provider schema validation and merging 
(#31974) (#31984)
cdc19d07da9 is described below

commit cdc19d07da928f8bf55bffa672a3a19e7941cf73
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Thu Jul 25 11:11:45 2024 -0400

[yaml] Fix yaml provider schema validation and merging (#31974) (#31984)

Signed-off-by: Jeffrey Kinard 
Co-authored-by: Jeff Kinard 
---
 sdks/python/apache_beam/yaml/pipeline.schema.yaml | 6 --
 sdks/python/apache_beam/yaml/yaml_transform.py| 4 ++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/sdks/python/apache_beam/yaml/pipeline.schema.yaml 
b/sdks/python/apache_beam/yaml/pipeline.schema.yaml
index f68a7306d94..c3937e61131 100644
--- a/sdks/python/apache_beam/yaml/pipeline.schema.yaml
+++ b/sdks/python/apache_beam/yaml/pipeline.schema.yaml
@@ -168,8 +168,10 @@ $defs:
 
   providerOrProviderInclude:
 if:
-  properties:
-include {}
+  allOf: [
+{ properties: { include: { type: string }}},
+{ required: [ "include" ] }
+  ]
 then:
   $ref: '#/$defs/providerInclude'
 else:
diff --git a/sdks/python/apache_beam/yaml/yaml_transform.py 
b/sdks/python/apache_beam/yaml/yaml_transform.py
index c1c509ebde2..ffef9bbcd8f 100644
--- a/sdks/python/apache_beam/yaml/yaml_transform.py
+++ b/sdks/python/apache_beam/yaml/yaml_transform.py
@@ -1064,5 +1064,5 @@ def expand_pipeline(
   return YamlTransform(
   pipeline_as_composite(pipeline_spec['pipeline']),
   yaml_provider.merge_providers(
-  pipeline_spec.get('providers', []), providers or
-  {})).expand(beam.pvalue.PBegin(pipeline))
+  yaml_provider.parse_providers(pipeline_spec.get('providers', [])),
+  providers or {})).expand(beam.pvalue.PBegin(pipeline))



(beam) branch release-2.58.0 updated: Dedup Lineage and getTableToExtract call in BigQuerySourceBase (#31960) (#31972)

2024-07-25 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/release-2.58.0 by this push:
 new 3ec56433f49 Dedup Lineage and getTableToExtract call in 
BigQuerySourceBase (#31960) (#31972)
3ec56433f49 is described below

commit 3ec56433f497ce2a9e217343de46e2bb99b252d9
Author: Yi Hu 
AuthorDate: Thu Jul 25 10:37:08 2024 -0400

Dedup Lineage and getTableToExtract call in BigQuerySourceBase (#31960) 
(#31972)
---
 .../java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java   | 3 ---
 1 file changed, 3 deletions(-)

diff --git 
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
 
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
index 38c0c8e43b2..998c82ab8d8 100644
--- 
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
+++ 
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
@@ -157,9 +157,6 @@ abstract class BigQuerySourceBase extends 
BoundedSource {
   LOG.info("Extract job produced {} files", res.extractedFiles.size());
   if (res.extractedFiles.size() > 0) {
 BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
-// emit this table ID as a lineage source
-Lineage.getSources()
-.add(BigQueryHelpers.dataCatalogName(getTableToExtract(bqOptions), 
bqOptions));
 final String extractDestinationDir =
 resolveTempLocation(bqOptions.getTempLocation(), 
"BigQueryExtractTemp", stepUuid);
 // Match all files in the destination directory to stat them in bulk.



(beam) branch master updated: Use buffered loggers that periodically flush. (#31977)

2024-07-25 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 496e7124272 Use buffered loggers that periodically flush. (#31977)
496e7124272 is described below

commit 496e7124272543ff08caa437e42d11fab99121d8
Author: tvalentyn 
AuthorDate: Thu Jul 25 07:22:02 2024 -0700

Use buffered loggers that periodically flush. (#31977)
---
 sdks/python/container/piputil.go | 13 ++---
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/sdks/python/container/piputil.go b/sdks/python/container/piputil.go
index 113bf405416..d6250ad2fdc 100644
--- a/sdks/python/container/piputil.go
+++ b/sdks/python/container/piputil.go
@@ -32,6 +32,7 @@ import (
"github.com/apache/beam/sdks/v2/go/pkg/beam/util/execx"
 )
 
+const pipLogFlushInterval time.Duration = 15 * time.Second
 const unrecoverableURL string = 
"https://beam.apache.org/documentation/sdks/python-unrecoverable-errors/index.html#pip-dependency-resolution-failures;
 
 // pipInstallRequirements installs the given requirement, if present.
@@ -40,7 +41,7 @@ func pipInstallRequirements(ctx context.Context, logger 
*tools.Logger, files []s
if err != nil {
return err
}
-   bufLogger := tools.NewBufferedLogger(logger)
+   bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger, 
pipLogFlushInterval)
for _, file := range files {
if file == name {
// We run the install process in two rounds in order to 
avoid as much
@@ -48,7 +49,7 @@ func pipInstallRequirements(ctx context.Context, logger 
*tools.Logger, files []s
// option will make sure that only things staged in the 
worker will be
// used without following their dependencies.
args := []string{"-m", "pip", "install", "-r", 
filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check", 
"--no-index", "--no-deps", "--find-links", dir}
-   if err := execx.Execute(pythonVersion, args...); err != 
nil {
+   if err := execx.ExecuteEnvWithIO(nil, os.Stdin, 
bufLogger, bufLogger, pythonVersion, args...); err != nil {
bufLogger.Printf(ctx, "Some packages could not 
be installed solely from the requirements cache. Installing packages from 
PyPI.")
}
// The second install round opens up the search for 
packages on PyPI and
@@ -79,8 +80,6 @@ func isPackageInstalled(pkgName string) bool {
return true
 }
 
-const pipLogFlushInterval time.Duration = 15 * time.Second
-
 // pipInstallPackage installs the given package, if present.
 func pipInstallPackage(ctx context.Context, logger *tools.Logger, files 
[]string, dir, name string, force, optional bool, extras []string) error {
pythonVersion, err := expansionx.GetPythonVersion()
@@ -150,7 +149,7 @@ func pipInstallPackage(ctx context.Context, logger 
*tools.Logger, files []string
 // installExtraPackages installs all the packages declared in the extra
 // packages manifest file.
 func installExtraPackages(ctx context.Context, logger *tools.Logger, files 
[]string, extraPackagesFile, dir string) error {
-   bufLogger := tools.NewBufferedLogger(logger)
+   bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger, 
pipLogFlushInterval)
// First check that extra packages manifest file is present.
for _, file := range files {
if file != extraPackagesFile {
@@ -179,7 +178,7 @@ func installExtraPackages(ctx context.Context, logger 
*tools.Logger, files []str
 }
 
 func findBeamSdkWhl(ctx context.Context, logger *tools.Logger, files []string, 
acceptableWhlSpecs []string) string {
-   bufLogger := tools.NewBufferedLogger(logger)
+   bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger, 
pipLogFlushInterval)
for _, file := range files {
if strings.HasPrefix(file, "apache_beam") {
for _, s := range acceptableWhlSpecs {
@@ -200,7 +199,7 @@ func findBeamSdkWhl(ctx context.Context, logger 
*tools.Logger, files []string, a
 // SDK from source tarball provided in sdkSrcFile.
 func installSdk(ctx context.Context, logger *tools.Logger, files []string, 
workDir string, sdkSrcFile string, acceptableWhlSpecs []string, required bool) 
error {
sdkWhlFile := findBeamSdkWhl(ctx, logger, files, acceptableWhlSpecs)
-   bufLogger := tools.NewBufferedLogger(logger)
+   bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger, 
pipLogFlushInterval)
if sdkWhlFile != "" {
// by default, pip rejects to install wheel if same version 
already installed
isDev := strings.Contains(sdkWhlFile, ".dev")



(beam) branch master updated: Note SpannerIO.read new validation in 2.58.0 breaking change section (#31973)

2024-07-25 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 8709f1126f6 Note SpannerIO.read new validation in 2.58.0 breaking 
change section (#31973)
8709f1126f6 is described below

commit 8709f1126f6c0975790ce7b5974a02429847de5f
Author: Yi Hu 
AuthorDate: Thu Jul 25 09:48:27 2024 -0400

Note SpannerIO.read new validation in 2.58.0 breaking change section 
(#31973)
---
 CHANGES.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGES.md b/CHANGES.md
index c4698e5585e..2bc2ecc4997 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -112,6 +112,7 @@
 
 * X behavior was changed ([#X](https://github.com/apache/beam/issues/X)).
 * [IcebergIO] IcebergCatalogConfig was changed to support specifying catalog 
properties in a key-store fashion 
([#31726](https://github.com/apache/beam/pull/31726))
+* [SpannerIO] Added validation that query and table cannot be specified at the 
same time for SpannerIO.read(). Previously withQuery overrides withTable, if 
set ([#24956](https://github.com/apache/beam/issues/24956)).
 
 ## Deprecations
 



(beam) branch release-2.58.0 updated: Use buffered loggers that periodically flush. (#31978)

2024-07-25 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/release-2.58.0 by this push:
 new 6e19e383972 Use buffered loggers that periodically flush. (#31978)
6e19e383972 is described below

commit 6e19e38397286c75847732d5f5cf52f235f03fc5
Author: tvalentyn 
AuthorDate: Thu Jul 25 06:46:11 2024 -0700

Use buffered loggers that periodically flush. (#31978)
---
 sdks/python/container/piputil.go | 13 ++---
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/sdks/python/container/piputil.go b/sdks/python/container/piputil.go
index 113bf405416..d6250ad2fdc 100644
--- a/sdks/python/container/piputil.go
+++ b/sdks/python/container/piputil.go
@@ -32,6 +32,7 @@ import (
"github.com/apache/beam/sdks/v2/go/pkg/beam/util/execx"
 )
 
+const pipLogFlushInterval time.Duration = 15 * time.Second
 const unrecoverableURL string = 
"https://beam.apache.org/documentation/sdks/python-unrecoverable-errors/index.html#pip-dependency-resolution-failures;
 
 // pipInstallRequirements installs the given requirement, if present.
@@ -40,7 +41,7 @@ func pipInstallRequirements(ctx context.Context, logger 
*tools.Logger, files []s
if err != nil {
return err
}
-   bufLogger := tools.NewBufferedLogger(logger)
+   bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger, 
pipLogFlushInterval)
for _, file := range files {
if file == name {
// We run the install process in two rounds in order to 
avoid as much
@@ -48,7 +49,7 @@ func pipInstallRequirements(ctx context.Context, logger 
*tools.Logger, files []s
// option will make sure that only things staged in the 
worker will be
// used without following their dependencies.
args := []string{"-m", "pip", "install", "-r", 
filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check", 
"--no-index", "--no-deps", "--find-links", dir}
-   if err := execx.Execute(pythonVersion, args...); err != 
nil {
+   if err := execx.ExecuteEnvWithIO(nil, os.Stdin, 
bufLogger, bufLogger, pythonVersion, args...); err != nil {
bufLogger.Printf(ctx, "Some packages could not 
be installed solely from the requirements cache. Installing packages from 
PyPI.")
}
// The second install round opens up the search for 
packages on PyPI and
@@ -79,8 +80,6 @@ func isPackageInstalled(pkgName string) bool {
return true
 }
 
-const pipLogFlushInterval time.Duration = 15 * time.Second
-
 // pipInstallPackage installs the given package, if present.
 func pipInstallPackage(ctx context.Context, logger *tools.Logger, files 
[]string, dir, name string, force, optional bool, extras []string) error {
pythonVersion, err := expansionx.GetPythonVersion()
@@ -150,7 +149,7 @@ func pipInstallPackage(ctx context.Context, logger 
*tools.Logger, files []string
 // installExtraPackages installs all the packages declared in the extra
 // packages manifest file.
 func installExtraPackages(ctx context.Context, logger *tools.Logger, files 
[]string, extraPackagesFile, dir string) error {
-   bufLogger := tools.NewBufferedLogger(logger)
+   bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger, 
pipLogFlushInterval)
// First check that extra packages manifest file is present.
for _, file := range files {
if file != extraPackagesFile {
@@ -179,7 +178,7 @@ func installExtraPackages(ctx context.Context, logger 
*tools.Logger, files []str
 }
 
 func findBeamSdkWhl(ctx context.Context, logger *tools.Logger, files []string, 
acceptableWhlSpecs []string) string {
-   bufLogger := tools.NewBufferedLogger(logger)
+   bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger, 
pipLogFlushInterval)
for _, file := range files {
if strings.HasPrefix(file, "apache_beam") {
for _, s := range acceptableWhlSpecs {
@@ -200,7 +199,7 @@ func findBeamSdkWhl(ctx context.Context, logger 
*tools.Logger, files []string, a
 // SDK from source tarball provided in sdkSrcFile.
 func installSdk(ctx context.Context, logger *tools.Logger, files []string, 
workDir string, sdkSrcFile string, acceptableWhlSpecs []string, required bool) 
error {
sdkWhlFile := findBeamSdkWhl(ctx, logger, files, acceptableWhlSpecs)
-   bufLogger := tools.NewBufferedLogger(logger)
+   bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger, 
pipLogFlushInterval)
if sdkWhlFile != "" {
// by default, pip rejects to install wheel if same version 
already installed
isDev := strings.Contains(sdkWhlFile, ".dev")



(beam) branch master updated: Auth with project id in our notebook (#31966)

2024-07-24 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new bf7f254 Auth with project id in our notebook (#31966)
bf7f254 is described below

commit bf7f2540f35bce23b251672575224e0762c1
Author: Danny McCormick 
AuthorDate: Wed Jul 24 17:15:58 2024 +0200

Auth with project id in our notebook (#31966)

* Auth with project id

* Wording/formatting
---
 .../beam-ml/bigtable_enrichment_transform.ipynb| 28 --
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/examples/notebooks/beam-ml/bigtable_enrichment_transform.ipynb 
b/examples/notebooks/beam-ml/bigtable_enrichment_transform.ipynb
index 7510831bac4..95be8b1d957 100644
--- a/examples/notebooks/beam-ml/bigtable_enrichment_transform.ipynb
+++ b/examples/notebooks/beam-ml/bigtable_enrichment_transform.ipynb
@@ -139,41 +139,33 @@
   },
   "source": [
 "### Authenticate with Google Cloud\n",
-"This notebook reads data from Pub/Sub and Bigtable. To use your 
Google Cloud account, authenticate this notebook."
+"This notebook reads data from Pub/Sub and Bigtable. To use your 
Google Cloud account, authenticate this notebook.\n",
+"To prepare for this step, replace ``, ``, 
and `` with the appropriate values for your setup. These fields are 
used with Bigtable."
   ]
 },
 {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
-"id": "Kz9sccyGBqz3"
+"id": "wEXucyi2liij"
   },
   "outputs": [],
   "source": [
-"from google.colab import auth\n",
-"auth.authenticate_user()"
-  ]
-},
-{
-  "cell_type": "markdown",
-  "metadata": {
-"id": "nAmGgUMt48o9"
-  },
-  "source": [
-"Replace ``, ``, and `` with the 
appropriate values for your setup. These fields are used with Bigtable."
+"PROJECT_ID = \"\"\n",
+"INSTANCE_ID = \"\"\n",
+"TABLE_ID = \"\""
   ]
 },
 {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
-"id": "wEXucyi2liij"
+"id": "Kz9sccyGBqz3"
   },
   "outputs": [],
   "source": [
-"PROJECT_ID = \"\"\n",
-"INSTANCE_ID = \"\"\n",
-"TABLE_ID = \"\""
+"from google.colab import auth\n",
+"auth.authenticate_user(project_id=PROJECT_ID)"
   ]
 },
 {
@@ -879,4 +871,4 @@
   },
   "nbformat": 4,
   "nbformat_minor": 0
-}
\ No newline at end of file
+}



(beam) branch master updated (8f2c72e8d22 -> 521586e04a7)

2024-07-18 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


from 8f2c72e8d22 Disable Dataflow run for java_test and python_test (#31934)
 add 521586e04a7 Update Python artifact name in release process (#31933)

No new revisions were added by this update.

Summary of changes:
 contributor-docs/release-guide.md | 12 +++
 release/src/main/scripts/run_rc_validation.sh | 48 +--
 2 files changed, 30 insertions(+), 30 deletions(-)



svn commit: r70374 - /dev/beam/2.58.0/python/

2024-07-17 Thread jrmccluskey
Author: jrmccluskey
Date: Wed Jul 17 19:40:23 2024
New Revision: 70374

Log:
Staging Python artifacts for Apache Beam 2.58.0 RC

Added:
dev/beam/2.58.0/python/

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-macosx_10_9_x86_64.whl   
(with props)

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-macosx_10_9_x86_64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-macosx_10_9_x86_64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
   (with props)

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl
   (with props)

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
   (with props)

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.sha512
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win32.whl   (with 
props)
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win32.whl.asc
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win32.whl.sha512
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win_amd64.whl   (with 
props)
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win_amd64.whl.asc
dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win_amd64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-macosx_10_9_x86_64.whl   
(with props)

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-macosx_10_9_x86_64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-macosx_10_9_x86_64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
   (with props)

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl
   (with props)

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
   (with props)

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.sha512
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win32.whl   (with 
props)
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win32.whl.asc
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win32.whl.sha512
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win_amd64.whl   (with 
props)
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win_amd64.whl.asc
dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win_amd64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-macosx_10_9_x86_64.whl   
(with props)

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-macosx_10_9_x86_64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-macosx_10_9_x86_64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
   (with props)

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl
   (with props)

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl.sha512

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
   (with props)

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc

dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_x86_64

(beam) annotated tag v2.58.0-RC1 updated (5b18d936cc0 -> 5ed765fbc2c)

2024-07-17 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag v2.58.0-RC1
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag v2.58.0-RC1 was modified! ***

from 5b18d936cc0 (commit)
  to 5ed765fbc2c (tag)
 tagging 5b18d936cc08507c3341c33e0977484b550298d0 (commit)
 replaces jupyterlab-sidepanel-v3.0.0
  by Jack McCluskey
  on Wed Jul 17 10:40:48 2024 -0400

- Log -
v2.58.0-RC1
---


No new revisions were added by this update.

Summary of changes:



(beam) annotated tag sdks/v2.58.0-RC1 updated (5b18d936cc0 -> a262768fd65)

2024-07-17 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC1
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag sdks/v2.58.0-RC1 was modified! ***

from 5b18d936cc0 (commit)
  to a262768fd65 (tag)
 tagging 5b18d936cc08507c3341c33e0977484b550298d0 (commit)
 replaces jupyterlab-sidepanel-v3.0.0
  by Jack McCluskey
  on Wed Jul 17 10:40:48 2024 -0400

- Log -
Go SDK v2.58.0-RC1
---


No new revisions were added by this update.

Summary of changes:



(beam) branch release-2.58.0 updated: Switch to use self-hosted runner for build_wheels action (#31866) (#31916)

2024-07-17 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/release-2.58.0 by this push:
 new eca36f69269 Switch to use self-hosted runner for build_wheels action 
(#31866) (#31916)
eca36f69269 is described below

commit eca36f6926967fae4f803baa4b1358d5faccab73
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Wed Jul 17 10:40:04 2024 -0400

Switch to use self-hosted runner for build_wheels action (#31866) (#31916)

* Switch to use self-hosted runner

* split os and runner string

Co-authored-by: Yi Hu 
---
 .github/workflows/build_wheels.yml | 52 --
 1 file changed, 16 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/build_wheels.yml 
b/.github/workflows/build_wheels.yml
index 5142b0b22c3..dc14359aa88 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -45,7 +45,7 @@ jobs:
   check_env_variables:
 timeout-minutes: 5
 name: "Check environment variables"
-runs-on: ubuntu-latest
+runs-on: [self-hosted, ubuntu-20.04, main]
 env:
   EVENT_NAME: ${{ github.event_name }}
   PY_VERSIONS_FULL: "cp38-* cp39-* cp310-* cp311-* cp312-*"
@@ -59,8 +59,8 @@ jobs:
 run: "./scripts/ci/ci_check_are_gcp_variables_set.sh"
 id: check_gcp_variables
 env:
-  GCP_SA_EMAIL: ${{ secrets.GCP_SA_EMAIL }}
-  GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+  GCP_SA_EMAIL: "not used by self hosted runner"
+  GCP_SA_KEY: "not used by self hosted runner"
   GCP_PYTHON_WHEELS_BUCKET: ${{ secrets.GCP_PYTHON_WHEELS_BUCKET }}
   GCP_PROJECT_ID: "not-needed-here"
   GCP_REGION: "not-needed-here"
@@ -80,7 +80,7 @@ jobs:
   echo "py-versions-full=$PY_VERSIONS_FULL" >> $GITHUB_OUTPUT
 
   build_source:
-runs-on: ubuntu-latest
+runs-on: [self-hosted, ubuntu-20.04, main]
 name: Build python source distribution
 outputs:
   is_rc: ${{ steps.is_rc.outputs.is_rc }}
@@ -190,14 +190,9 @@ jobs:
 needs:
   - build_source
   - check_env_variables
-runs-on: ubuntu-latest
+runs-on: [self-hosted, ubuntu-20.04, main]
 if: needs.check_env_variables.outputs.gcp-variables-set == 'true' && 
github.event_name != 'pull_request'
 steps:
-  - name: Authenticate on GCP
-uses: google-github-actions/setup-gcloud@v0
-with:
-  service_account_email: ${{ secrets.GCP_SA_EMAIL }}
-  service_account_key: ${{ secrets.GCP_SA_KEY }}
   - name: Remove existing files on GCS bucket
 run: gsutil rm -r ${{ env.GCP_PATH }} || true
 
@@ -206,7 +201,7 @@ jobs:
 needs:
   - prepare_gcs
   - check_env_variables
-runs-on: ubuntu-latest
+runs-on: [self-hosted, ubuntu-20.04, main]
 if: needs.check_env_variables.outputs.gcp-variables-set == 'true'
 steps:
   - name: Download compressed sources from artifacts
@@ -215,11 +210,6 @@ jobs:
 with:
   name: source_zip
   path: source/
-  - name: Authenticate on GCP
-uses: google-github-actions/setup-gcloud@v0
-with:
-  service_account_email: ${{ secrets.GCP_SA_EMAIL }}
-  service_account_key: ${{ secrets.GCP_SA_KEY }}
   - name: Copy sources to GCS bucket
 run: gsutil cp -r -a public-read source/* ${{ env.GCP_PATH }}
 
@@ -230,19 +220,19 @@ jobs:
   - build_source
 env:
   CIBW_ARCHS_LINUX: ${{matrix.arch}}
-runs-on: ${{ matrix.os_python.os }}
+runs-on: ${{ matrix.os_python.runner }}
 strategy:
   matrix:
 os_python: [
-  {"os": "ubuntu-latest", "python": "${{ 
needs.check_env_variables.outputs.py-versions-full }}" },
+  {"os": "ubuntu-20.04", "runner": [self-hosted, ubuntu-20.04, main], 
"python": "${{ needs.check_env_variables.outputs.py-versions-full }}" },
   # Temporarily pin to macos-13 because macos-latest breaks this build
   # TODO(https://github.com/apache/beam/issues/31114)
-  {"os": "macos-13", "python": "${{ 
needs.check_env_variables.outputs.py-versions-test }}" },
-  {"os": "windows-latest", "python": "${{ 
needs.check_env_variables.outputs.py-versions-test }}" },
+  {"os": "macos-13", "runner": "macos-13", "python": "${{ 
needs.check_env_variables.outputs.py-versions-test }}" },
+  {"os": "windows-latest", "runner": &quo

(beam) annotated tag sdks/v2.58.0-RC1 deleted (was e264a659ecd)

2024-07-17 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC1
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag sdks/v2.58.0-RC1 was deleted! ***

   tag was  e264a659ecd

This change permanently discards the following revisions:

 discard 06ef4974894 Set version for 2.58.0 RC1



(beam) annotated tag v2.58.0-RC1 deleted (was d1da3b2a5db)

2024-07-17 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag v2.58.0-RC1
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag v2.58.0-RC1 was deleted! ***

   tag was  d1da3b2a5db

The revisions that were on this annotated tag are still contained in
other references; therefore, this change does not discard any commits
from the repository.



(beam-site) branch updates_release_2.58.0 created (now c9292ee7bc)

2024-07-16 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch updates_release_2.58.0
in repository https://gitbox.apache.org/repos/asf/beam-site.git


  at c9292ee7bc Update beam-site for release 2.58.0.

No new revisions were added by this update.



svn commit: r70354 - in /dev/beam/2.58.0/prism: ./ darwin/ darwin/amd64/ darwin/arm64/ linux/ linux/amd64/ linux/arm64/ windows/ windows/amd64/ windows/arm64/

2024-07-16 Thread jrmccluskey
Author: jrmccluskey
Date: Tue Jul 16 21:07:04 2024
New Revision: 70354

Log:
Staging Prism artifacts for Apache Beam 2.58.0 RC

Added:
dev/beam/2.58.0/prism/
dev/beam/2.58.0/prism/darwin/
dev/beam/2.58.0/prism/darwin/amd64/

dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip   
(with props)

dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc

dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512
dev/beam/2.58.0/prism/darwin/arm64/

dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip   
(with props)

dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip.asc

dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip.sha512
dev/beam/2.58.0/prism/linux/
dev/beam/2.58.0/prism/linux/amd64/
dev/beam/2.58.0/prism/linux/amd64/apache_beam-v2.58.0-prism-linux-amd64.zip 
  (with props)

dev/beam/2.58.0/prism/linux/amd64/apache_beam-v2.58.0-prism-linux-amd64.zip.asc

dev/beam/2.58.0/prism/linux/amd64/apache_beam-v2.58.0-prism-linux-amd64.zip.sha512
dev/beam/2.58.0/prism/linux/arm64/
dev/beam/2.58.0/prism/linux/arm64/apache_beam-v2.58.0-prism-linux-arm64.zip 
  (with props)

dev/beam/2.58.0/prism/linux/arm64/apache_beam-v2.58.0-prism-linux-arm64.zip.asc

dev/beam/2.58.0/prism/linux/arm64/apache_beam-v2.58.0-prism-linux-arm64.zip.sha512
dev/beam/2.58.0/prism/windows/
dev/beam/2.58.0/prism/windows/amd64/

dev/beam/2.58.0/prism/windows/amd64/apache_beam-v2.58.0-prism-windows-amd64.zip 
  (with props)

dev/beam/2.58.0/prism/windows/amd64/apache_beam-v2.58.0-prism-windows-amd64.zip.asc

dev/beam/2.58.0/prism/windows/amd64/apache_beam-v2.58.0-prism-windows-amd64.zip.sha512
dev/beam/2.58.0/prism/windows/arm64/

dev/beam/2.58.0/prism/windows/arm64/apache_beam-v2.58.0-prism-windows-arm64.zip 
  (with props)

dev/beam/2.58.0/prism/windows/arm64/apache_beam-v2.58.0-prism-windows-arm64.zip.asc

dev/beam/2.58.0/prism/windows/arm64/apache_beam-v2.58.0-prism-windows-arm64.zip.sha512

Added: 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip
==
Binary file - no diff available.

Propchange: 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip
--
svn:mime-type = application/octet-stream

Added: 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc
==
--- 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc
 (added)
+++ 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc
 Tue Jul 16 21:07:04 2024
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+
+iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaW4MMYHHByaXZhdGVA
+YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiy5oP/RmV0UdqiUOSuoHy7WhrJEle
+xfW0HIiXguSAUCxMmVOoVbtuHk8lmBcpXkQqbDvTqoaPZbLqFST3nFpf8hxeAmCW
+mjiNyZ5fkYjjLVPR7BftjGNzRBoHsjX0B28hwsgA26O2BT7CJdXtGT31xzg413hK
+WFoI3UEAkJV1LiSkePCFKmnoPIN+642GztD+LQ7KB32S/Va33i1RjNNS1Wt5mhSh
+Rv5gdOCdXBk73nm5Sfvk9PREE6Q3nJzP9ts4Nng0QVJNaXfz4mcWl4UiGbdKr/8d
+pAYaKABse5bxL+MStsjj0ZcshUIHHPN9Xs/qqguzfGSmMFFvw88fHYS4xfg/XuKk
+BMCokNFfkUrTq74j02k13CTvvayqjuyu/l42sn7W2r0M+6XoTtivHSruSvJechLc
+NfiJ4DrUctLAfk133LiCmP0j76kpk6gq5RdMlRSPq9Dz3deE0VcF0egTH/mfUMOa
+tUP1VWru8IO0wmfWX+qNM+KpZ+U28bFYpPb4Vzx2BljSgXtPk8BQmciyjw5TVEQd
+M458oh+rHU+T9zx8n+TFkGmodyx0/0khmq8LbrRsNGaPNiZs98QH9TfTe7ahYTht
+3QirvJqAKQES1GYGNlZI30dH3v1rJvgo4IJ9TkuFc3pFxUe5gViVpb73LQRQf0CX
+V23uewV1pHiXXd5vUDe+
+=Z3/h
+-END PGP SIGNATURE-

Added: 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512
==
--- 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512
 (added)
+++ 
dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512
 Tue Jul 16 21:07:04 2024
@@ -0,0 +1 @@
+a76dc4a051dbbaee8e85073b56921302dce28f619d91be532d4a3b80183d666a9d53cf0f1d01df2dc962e9f7501775f9af0ac29a9104ab7b03223f239ea0605a
  apache_beam-v2.58.0-prism-darwin-amd64.zip

Added: 
dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip
==
Binary file - no diff available.

Propchange: 
dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip
--
svn:mime-type = application/octet-stream

Added: 
dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip.asc

svn commit: r70353 - in /dev/beam/2.58.0: apache-beam-2.58.0-source-release.zip apache-beam-2.58.0-source-release.zip.asc apache-beam-2.58.0-source-release.zip.sha512

2024-07-16 Thread jrmccluskey
Author: jrmccluskey
Date: Tue Jul 16 20:59:39 2024
New Revision: 70353

Log:
Staging Java artifacts for Apache Beam 2.58.0 RC1

Added:
dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip   (with props)
Modified:
dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc
dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512

Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip
==
Binary file - no diff available.

Propchange: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip
--
svn:mime-type = application/octet-stream

Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc
==
--- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc (original)
+++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc Tue Jul 16 
20:59:39 2024
@@ -1,17 +1,17 @@
 -BEGIN PGP SIGNATURE-
 
-iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaWv9QYHHByaXZhdGVA
-YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiQyIQAMHFAKjHh66owHM98d8tLZXv
-x6JpzyILoWu36GahKOXBT1pCqyXguoKm01L/EpCBYFEcoICY8q4KaVgoQZvefRDe
-/8c7pkaVXX25lbnfiKdzur2sXi6dsN8LP3P41yo9y7V6x+moWyMr/GcnsWPUBs6y
-LDnp7DL2E+EhFpWyO4Iey+KG3KH8jttbGKTgGPcuh3aYFjsGpQRKsHsgYXIR3fAP
-e4kK8dp3AZ26oHUIaLfH9HyxqLUB1HxqYViealy4AnIS6Q6AuoPhBi4oeFegyl+m
-ZUNEX2eWXf9gA9G77SzeS3UF54NN+8SRbPabG7wvnRiyUGAYwcvug+bnhtYhrD02
-CRePK0ZLLCUyYVojVofvTLV/3fu5Qv+5jWudn48wWH2bNJn3fhbmFuf4qHm4OXgW
-bWUmSbslQXuqmVFZbDXYYwplrG1cH55rR/ydwSoobvfiybUPu+O9ltPgQLTS1Hxh
-xDW1zKnioXJm2UZik7f+62c5B+pfIWQ9mDPN4h+Jlj9PJDM1KWfCRx9NoWsvKlkc
-g5Oo0xul+qF4+ER78h/+0bKhVNbkdo2nGsxPB9Di1LIyzkcLs5H8vHJB/un6IZjl
-1gT4UDNdfr6ZHKb8MKMrAFyEx0kuXsDHXa405gZgRNNNQGbq2kUcFZP6thIERlgS
-gvLNAdj8pQbw+bzRNdAR
-=bT/s
+iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaW3zMYHHByaXZhdGVA
+YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQin6MP/3mX5HNCLLAheR94p5jWsobu
+UduIN/nrXKYOcDAuulm9bgOAfQgJh/4P8Ds60uf5vFCYtPGWwR3SFOf3jBFgn71c
+A990YYS2uU1vCkgI1hHASHNaViV6vuTUre2qUK5nZDRr5GH66GxIQPsIVy/mKojx
+9A7gaAueKTG4EmqF5zYJr23vk6ZKEGVBV8mjpsZ5MNsV0NmKtHuzuTX9o4JPAcVa
+8jHfyeRfLnzDhLEeGTa30NbN2ofk4YnDrHgsyBfb3DXyBZd5nGxpPTexAH3ehJG7
+XPasWTCo/024XSQwfEhkAcYSCn0t+XgJe/axq+VPC2E9Sdp2lr+MNudYlmefBUIL
+YvMM0AazTuYJajJQCBho7Zx5W1aYY1CFM73Af9N/9Fs9i5KfcPeyKvm9ehDC+CD6
+HOXzejGtPBJ3T44DWz/Im+2c6mktDAXmd6adNTw20uxGDYMhESI08cA00pxwTTGA
+NLLk0uAQ+QGB0YaLRH7svbZt41pHSTNyVlZWNrKYYGt5Kce3pWGTkyIJc0MBWpII
+pS9UmzMm1sjcogDnGcTWl78VLCQby9kgzKyB6DBAJbh63CAUkR/k8jhXqXVz6Q1p
+JChap4jSLn0GKnHtSrE1XDXcTUkd/EVu3ASr8564QdpQbhs3nKk6luwtp8DaEyg3
+HA7cTMaUYVsRu1hSdrQL
+=zDx7
 -END PGP SIGNATURE-

Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512
==
--- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 (original)
+++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 Tue Jul 16 
20:59:39 2024
@@ -1 +1 @@
-845a828fefeee560fb6b2d9bc318cc4a7c4682e1410053f7b5a03064becba40a6d585eb453b1a2af0a04563b95ae441d59c317ec255f86dc65ad7e191b9545af
  apache-beam-2.58.0-source-release.zip
+48b6f6240337f5ea96c7bc9b44de93d581493ff5a4e78006e1c514810853ba972bc9325073b635b16b32c88fbdda8c4f87cb822e8e03de8ebd4cd5fc4254de4f
  apache-beam-2.58.0-source-release.zip




(beam) annotated tag v2.58.0-RC1 updated (06ef4974894 -> d1da3b2a5db)

2024-07-16 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag v2.58.0-RC1
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag v2.58.0-RC1 was modified! ***

from 06ef4974894 (commit)
  to d1da3b2a5db (tag)
 tagging 06ef49748946e9bfef6fe28332089a39aff7fbe7 (commit)
 replaces jupyterlab-sidepanel-v3.0.0
  by Jack McCluskey
  on Tue Jul 16 16:51:06 2024 -0400

- Log -
v2.58.0-RC1
---


No new revisions were added by this update.

Summary of changes:



(beam) annotated tag sdks/v2.58.0-RC1 updated (06ef4974894 -> e264a659ecd)

2024-07-16 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC1
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag sdks/v2.58.0-RC1 was modified! ***

from 06ef4974894 (commit)
  to e264a659ecd (tag)
 tagging 06ef49748946e9bfef6fe28332089a39aff7fbe7 (commit)
 replaces jupyterlab-sidepanel-v3.0.0
  by Jack McCluskey
  on Tue Jul 16 16:51:06 2024 -0400

- Log -
Go SDK v2.58.0-RC1
---


No new revisions were added by this update.

Summary of changes:



(beam) branch release-2.58.0 updated: Fix generateYamlDocs gradle task (#31909)

2024-07-16 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/release-2.58.0 by this push:
 new 7626ad83d4e Fix generateYamlDocs gradle task (#31909)
7626ad83d4e is described below

commit 7626ad83d4e9182a5557ac52836c032b82f12e10
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Tue Jul 16 16:23:14 2024 -0400

Fix generateYamlDocs gradle task (#31909)
---
 sdks/python/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index 72c696d30bf..832f567c66f 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -95,7 +95,7 @@ tasks.register("generateYamlDocs") {
   dependsOn ":sdks:java:extensions:sql:expansion-service:shadowJar"
   dependsOn ":sdks:java:io:expansion-service:build"
   dependsOn ":sdks:java:io:google-cloud-platform:expansion-service:build"
-  def extraPackages = "pyyaml markdown docstring_parser pandas pygments"
+  def extraPackages = "pyyaml markdown docstring_parser pandas pygments Jinja2"
 
   doLast {
 exec {



(beam) branch master updated: Fix generateYamlDocs gradle task (#31909)

2024-07-16 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 2d6d55b98ce Fix generateYamlDocs gradle task (#31909)
2d6d55b98ce is described below

commit 2d6d55b98cef5eacc4abd93863de2132cf89bea2
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Tue Jul 16 16:23:14 2024 -0400

Fix generateYamlDocs gradle task (#31909)
---
 sdks/python/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index 72c696d30bf..832f567c66f 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -95,7 +95,7 @@ tasks.register("generateYamlDocs") {
   dependsOn ":sdks:java:extensions:sql:expansion-service:shadowJar"
   dependsOn ":sdks:java:io:expansion-service:build"
   dependsOn ":sdks:java:io:google-cloud-platform:expansion-service:build"
-  def extraPackages = "pyyaml markdown docstring_parser pandas pygments"
+  def extraPackages = "pyyaml markdown docstring_parser pandas pygments Jinja2"
 
   doLast {
 exec {



(beam) annotated tag sdks/v2.58.0-RC1 deleted (was f39c3f5f778)

2024-07-16 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC1
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag sdks/v2.58.0-RC1 was deleted! ***

   tag was  f39c3f5f778

This change permanently discards the following revisions:

 discard b6aba7d9b97 Set version for 2.58.0 RC1



(beam) annotated tag v2.58.0-RC1 deleted (was 4a804951223)

2024-07-16 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag v2.58.0-RC1
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag v2.58.0-RC1 was deleted! ***

   tag was  4a804951223

This change permanently discards the following revisions:

 discard b6aba7d9b97 Set version for 2.58.0 RC1



svn commit: r70349 - in /dev/beam/2.58.0: ./ apache-beam-2.58.0-source-release.zip apache-beam-2.58.0-source-release.zip.asc apache-beam-2.58.0-source-release.zip.sha512

2024-07-16 Thread jrmccluskey
Author: jrmccluskey
Date: Tue Jul 16 18:45:46 2024
New Revision: 70349

Log:
Staging Java artifacts for Apache Beam 2.58.0 RC1

Added:
dev/beam/2.58.0/
dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip   (with props)
dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc
dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512

Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip
==
Binary file - no diff available.

Propchange: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip
--
svn:mime-type = application/octet-stream

Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc
==
--- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc (added)
+++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc Tue Jul 16 
18:45:46 2024
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+
+iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaWv9QYHHByaXZhdGVA
+YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiQyIQAMHFAKjHh66owHM98d8tLZXv
+x6JpzyILoWu36GahKOXBT1pCqyXguoKm01L/EpCBYFEcoICY8q4KaVgoQZvefRDe
+/8c7pkaVXX25lbnfiKdzur2sXi6dsN8LP3P41yo9y7V6x+moWyMr/GcnsWPUBs6y
+LDnp7DL2E+EhFpWyO4Iey+KG3KH8jttbGKTgGPcuh3aYFjsGpQRKsHsgYXIR3fAP
+e4kK8dp3AZ26oHUIaLfH9HyxqLUB1HxqYViealy4AnIS6Q6AuoPhBi4oeFegyl+m
+ZUNEX2eWXf9gA9G77SzeS3UF54NN+8SRbPabG7wvnRiyUGAYwcvug+bnhtYhrD02
+CRePK0ZLLCUyYVojVofvTLV/3fu5Qv+5jWudn48wWH2bNJn3fhbmFuf4qHm4OXgW
+bWUmSbslQXuqmVFZbDXYYwplrG1cH55rR/ydwSoobvfiybUPu+O9ltPgQLTS1Hxh
+xDW1zKnioXJm2UZik7f+62c5B+pfIWQ9mDPN4h+Jlj9PJDM1KWfCRx9NoWsvKlkc
+g5Oo0xul+qF4+ER78h/+0bKhVNbkdo2nGsxPB9Di1LIyzkcLs5H8vHJB/un6IZjl
+1gT4UDNdfr6ZHKb8MKMrAFyEx0kuXsDHXa405gZgRNNNQGbq2kUcFZP6thIERlgS
+gvLNAdj8pQbw+bzRNdAR
+=bT/s
+-END PGP SIGNATURE-

Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512
==
--- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 (added)
+++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 Tue Jul 16 
18:45:46 2024
@@ -0,0 +1 @@
+845a828fefeee560fb6b2d9bc318cc4a7c4682e1410053f7b5a03064becba40a6d585eb453b1a2af0a04563b95ae441d59c317ec255f86dc65ad7e191b9545af
  apache-beam-2.58.0-source-release.zip




(beam) annotated tag v2.58.0-RC1 updated (b6aba7d9b97 -> 4a804951223)

2024-07-16 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag v2.58.0-RC1
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag v2.58.0-RC1 was modified! ***

from b6aba7d9b97 (commit)
  to 4a804951223 (tag)
 tagging b6aba7d9b977fb881eaeb2c3127eaf6984dad01d (commit)
 replaces jupyterlab-sidepanel-v3.0.0
  by jrmccluskey
  on Tue Jul 16 18:35:34 2024 +

- Log -
v2.58.0-RC1
---


No new revisions were added by this update.

Summary of changes:



(beam) annotated tag sdks/v2.58.0-RC1 updated (b6aba7d9b97 -> f39c3f5f778)

2024-07-16 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC1
in repository https://gitbox.apache.org/repos/asf/beam.git


*** WARNING: tag sdks/v2.58.0-RC1 was modified! ***

from b6aba7d9b97 (commit)
  to f39c3f5f778 (tag)
 tagging b6aba7d9b977fb881eaeb2c3127eaf6984dad01d (commit)
 replaces jupyterlab-sidepanel-v3.0.0
  by jrmccluskey
  on Tue Jul 16 18:35:34 2024 +

- Log -
Go SDK v2.58.0-RC1
---


No new revisions were added by this update.

Summary of changes:



(beam) branch release-2.58.0 updated: Export string sets in monitoring infos. (#31838) (#31849)

2024-07-16 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/release-2.58.0 by this push:
 new 92de083fd98 Export string sets in monitoring infos. (#31838) (#31849)
92de083fd98 is described below

commit 92de083fd98445eb993a2c708a835b874a91f765
Author: Robert Bradshaw 
AuthorDate: Tue Jul 16 10:45:01 2024 -0700

Export string sets in monitoring infos. (#31838) (#31849)
---
 .../apache/beam/model/pipeline/v1/metrics.proto| 11 +
 .../runners/core/metrics/MetricsContainerImpl.java | 53 +-
 .../core/metrics/MonitoringInfoConstants.java  |  2 +
 .../core/metrics/SimpleMonitoringInfoBuilder.java  | 11 +
 .../core/metrics/MetricsContainerImplTest.java | 32 +
 5 files changed, 108 insertions(+), 1 deletion(-)

diff --git 
a/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/metrics.proto 
b/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/metrics.proto
index 13c87bc1130..4ec189e4637 100644
--- 
a/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/metrics.proto
+++ 
b/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/metrics.proto
@@ -187,6 +187,17 @@ message MonitoringInfoSpecs {
   }]
 }];
 
+// Represents a set of strings seen across bundles.
+USER_SET_STRING = 21 [(monitoring_info_spec) = {
+  urn: "beam:metric:user:set_string:v1",
+  type: "beam:metrics:set_string:v1",
+  required_labels: ["PTRANSFORM", "NAMESPACE", "NAME"],
+  annotations: [{
+key: "description",
+value: "URN utilized to report user metric."
+  }]
+}];
+
 // General monitored state information which contains structured 
information
 // which does not fit into a typical metric format. See MonitoringTableData
 // for more details.
diff --git 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
index a2f6511d512..99cf9850850 100644
--- 
a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
+++ 
b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
@@ -19,13 +19,16 @@ package org.apache.beam.runners.core.metrics;
 
 import static 
org.apache.beam.runners.core.metrics.MonitoringInfoConstants.TypeUrns.DISTRIBUTION_INT64_TYPE;
 import static 
org.apache.beam.runners.core.metrics.MonitoringInfoConstants.TypeUrns.LATEST_INT64_TYPE;
+import static 
org.apache.beam.runners.core.metrics.MonitoringInfoConstants.TypeUrns.SET_STRING_TYPE;
 import static 
org.apache.beam.runners.core.metrics.MonitoringInfoConstants.TypeUrns.SUM_INT64_TYPE;
 import static 
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.decodeInt64Counter;
 import static 
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.decodeInt64Distribution;
 import static 
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.decodeInt64Gauge;
+import static 
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.decodeStringSet;
 import static 
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeInt64Counter;
 import static 
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeInt64Distribution;
 import static 
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeInt64Gauge;
+import static 
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeStringSet;
 import static 
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
 
 import java.io.Serializable;
@@ -331,6 +334,28 @@ public class MetricsContainerImpl implements Serializable, 
MetricsContainer {
 return builder.build();
   }
 
+  /** @return The MonitoringInfo metadata from the string set metric. */
+  private @Nullable SimpleMonitoringInfoBuilder 
stringSetToMonitoringMetadata(MetricKey metricKey) {
+return metricToMonitoringMetadata(
+metricKey,
+MonitoringInfoConstants.TypeUrns.SET_STRING_TYPE,
+MonitoringInfoConstants.Urns.USER_SET_STRING);
+  }
+
+  /**
+   * @param metricUpdate
+   * @return The MonitoringInfo generated from the string set metricUpdate.
+   */
+  private @Nullable MonitoringInfo stringSetUpdateToMonitoringInfo(
+  MetricUpdate metricUpdate) {
+SimpleMonitoringInfoBuilder builder = 
stringSetToMonitoringMetadata(metricUpdate.getKey());
+if (builder == null) {
+  return null;
+}
+builder.setStringSetValue(metricUpdate.getUpdate());
+return builder.build();
+  }
+
   /** Return the cumulative values for any metrics in this container as 
MonitoringInfos. */
   @Override
   public Iterable getMoni

(beam) branch release-2.58.0 updated (fb96d6f42f1 -> dea262378cd)

2024-07-16 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git


from fb96d6f42f1 Cherrypicking #31721 to release 2.58.0 (#31868)
 add dea262378cd Cherrypicking #31837 (#31904)

No new revisions were added by this update.

Summary of changes:
 CHANGES.md | 3 ++-
 .../beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java  | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)



(beam) branch release-2.58.0 updated: Cherrypicking #31721 to release 2.58.0 (#31868)

2024-07-15 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/release-2.58.0 by this push:
 new fb96d6f42f1 Cherrypicking #31721 to release 2.58.0 (#31868)
fb96d6f42f1 is described below

commit fb96d6f42f1b6f49ac2d35f61e9cca524d497877
Author: Ahmed Abualsaud <65791736+ahmedab...@users.noreply.github.com>
AuthorDate: Mon Jul 15 10:48:52 2024 -0400

Cherrypicking #31721 to release 2.58.0 (#31868)

* add options to set min and max connections to connection management pool; 
rename counter to be more accurate

* add multiplexing description

* add to CHANGES.md

* whitespace

* doc

* clarify documentation and address comments

* adjust description

* add details
---
 CHANGES.md |  2 ++
 .../beam/sdk/io/gcp/bigquery/AppendClientInfo.java |  8 +++
 .../beam/sdk/io/gcp/bigquery/BigQueryOptions.java  | 27 ++
 .../sdk/io/gcp/bigquery/BigQueryServicesImpl.java  |  9 
 4 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 96c436d89ec..fc94877a2bb 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -67,6 +67,7 @@
 ## New Features / Improvements
 
 * Multiple RunInference instances can now share the same model instance by 
setting the model_identifier parameter (Python) 
([#31665](https://github.com/apache/beam/issues/31665)).
+* Added options to control the number of Storage API multiplexing connections 
([#31721](https://github.com/apache/beam/pull/31721))
 * [IcebergIO] All specified catalog properties are passed through to the 
connector ([#31726](https://github.com/apache/beam/pull/31726))
 * Removed a 3rd party LGPL dependency from the Go SDK 
([#31765](https://github.com/apache/beam/issues/31765)).
 * Support for MapState and SetState when using Dataflow Runner v1 with 
Streaming Engine (Java) 
([[#18200](https://github.com/apache/beam/issues/18200)])
@@ -82,6 +83,7 @@
 
 ## Bugfixes
 
+* Fixed a bug in BigQueryIO batch Storage Write API that frequently exhausted 
concurrent connections quota 
([#31710](https://github.com/apache/beam/pull/31710))
 * Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
 
 ## Security Fixes
diff --git 
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java
 
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java
index 5a12e81ea79..7505f77fb5b 100644
--- 
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java
+++ 
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java
@@ -40,8 +40,8 @@ import org.apache.beam.sdk.metrics.Metrics;
  */
 @AutoValue
 abstract class AppendClientInfo {
-  private final Counter activeConnections =
-  Metrics.counter(AppendClientInfo.class, "activeConnections");
+  private final Counter activeStreamAppendClients =
+  Metrics.counter(AppendClientInfo.class, "activeStreamAppendClients");
 
   abstract @Nullable BigQueryServices.StreamAppendClient 
getStreamAppendClient();
 
@@ -123,7 +123,7 @@ abstract class AppendClientInfo {
   writeStreamService.getStreamAppendClient(
   streamName, getDescriptor(), useConnectionPool, 
missingValueInterpretation);
 
-  activeConnections.inc();
+  activeStreamAppendClients.inc();
 
   return 
toBuilder().setStreamName(streamName).setStreamAppendClient(client).build();
 }
@@ -133,7 +133,7 @@ abstract class AppendClientInfo {
 BigQueryServices.StreamAppendClient client = getStreamAppendClient();
 if (client != null) {
   getCloseAppendClient().accept(client);
-  activeConnections.dec();
+  activeStreamAppendClients.dec();
 }
   }
 
diff --git 
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java
 
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java
index cd1fc6d3842..ba76f483f77 100644
--- 
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java
+++ 
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java
@@ -109,6 +109,28 @@ public interface BigQueryOptions
 
   void setNumStorageWriteApiStreamAppendClients(Integer value);
 
+  @Description(
+  "When using the STORAGE_API_AT_LEAST_ONCE write method with multiplexing 
(ie. useStorageApiConnectionPool=true), "
+  + "this option sets the minimum number of connections each pool 
creates before any connections are shared. This is "
+  + "on a per worker, pe

(beam) branch release-2.58.0 updated (b95a750e862 -> d5a9bbdfe3c)

2024-07-11 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git


from b95a750e862 Merge pull request #31823 Add lineage information for 
BigQuery sinks. (#31836)
 add d5a9bbdfe3c Change orphan file log to warning (#31845)

No new revisions were added by this update.

Summary of changes:
 sdks/python/apache_beam/io/fileio.py | 14 +++---
 1 file changed, 7 insertions(+), 7 deletions(-)



(beam) branch release-2.58.0 updated: Merge pull request #31823 Add lineage information for BigQuery sinks. (#31836)

2024-07-10 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/release-2.58.0 by this push:
 new b95a750e862 Merge pull request #31823 Add lineage information for 
BigQuery sinks. (#31836)
b95a750e862 is described below

commit b95a750e862d5f9a179805528eff5dd0b605f597
Author: Robert Bradshaw 
AuthorDate: Wed Jul 10 13:50:55 2024 -0700

Merge pull request #31823 Add lineage information for BigQuery sinks. 
(#31836)
---
 .../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java  | 21 +++--
 .../sdk/io/gcp/bigquery/BigQuerySourceBase.java|  5 ++--
 .../io/gcp/bigquery/BigQueryStorageSourceBase.java |  4 +--
 .../beam/sdk/io/gcp/bigquery/CreateTables.java |  8 -
 .../bigquery/StorageApiWriteUnshardedRecords.java  | 14 +
 .../bigquery/StorageApiWritesShardedRecords.java   |  6 
 .../beam/sdk/io/gcp/bigquery/WriteRename.java  |  6 
 .../beam/sdk/io/gcp/bigquery/WriteTables.java  |  6 
 .../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java   | 34 +++---
 9 files changed, 92 insertions(+), 12 deletions(-)

diff --git 
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
 
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
index 7f5d675ccf7..61bed66a336 100644
--- 
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
+++ 
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -54,6 +54,7 @@ import 
org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.util.FluentBackoff;
 import 
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings;
 import 
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists;
 import org.checkerframework.checker.nullness.qual.NonNull;
 import org.checkerframework.checker.nullness.qual.Nullable;
@@ -412,9 +413,23 @@ public class BigQueryHelpers {
 return sb.toString();
   }
 
-  public static String dataCatalogName(TableReference ref) {
-return String.format(
-"bigquery:%s.%s.%s", ref.getProjectId(), ref.getDatasetId(), 
ref.getTableId());
+  public static String dataCatalogName(TableReference ref, BigQueryOptions 
options) {
+String tableIdBase;
+int ix = ref.getTableId().indexOf('$');
+if (ix == -1) {
+  tableIdBase = ref.getTableId();
+} else {
+  tableIdBase = ref.getTableId().substring(0, ix);
+}
+String projectId;
+if (!Strings.isNullOrEmpty(ref.getProjectId())) {
+  projectId = ref.getProjectId();
+} else if (!Strings.isNullOrEmpty(options.getBigQueryProject())) {
+  projectId = options.getBigQueryProject();
+} else {
+  projectId = options.getProject();
+}
+return String.format("bigquery:%s.%s.%s", projectId, ref.getDatasetId(), 
tableIdBase);
   }
 
   static  List getOrCreateMapListValue(Map> map, K key) {
diff --git 
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
 
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
index a863c49f46a..38c0c8e43b2 100644
--- 
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
+++ 
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
@@ -121,7 +121,7 @@ abstract class BigQuerySourceBase extends 
BoundedSource {
 BigQueryHelpers.toTableSpec(tableToExtract)));
   }
   // emit this table ID as a lineage source
-  
Lineage.getSources().add(BigQueryHelpers.dataCatalogName(tableToExtract));
+  Lineage.getSources().add(BigQueryHelpers.dataCatalogName(tableToExtract, 
bqOptions));
 
   TableSchema schema = table.getSchema();
   JobService jobService = bqServices.getJobService(bqOptions);
@@ -158,7 +158,8 @@ abstract class BigQuerySourceBase extends 
BoundedSource {
   if (res.extractedFiles.size() > 0) {
 BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
 // emit this table ID as a lineage source
-
Lineage.getSources().add(BigQueryHelpers.dataCatalogName(getTableToExtract(bqOptions)));
+Lineage.getSources()
+.add(BigQueryHelpers.dataCatalogName(getTableToExtract(bqOptions), 
bqOptions));
 final String extractDestinationDir =
 resolveTempLocation(bqOptions.getTempLocation(), 
"BigQueryExtractTemp", stepUuid);
 /

(beam) branch master updated (b9a44126622 -> 6c829db657b)

2024-07-10 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


from b9a44126622 Add changelog notes regarding Solace read connector 
(#31826)
 add 6c829db657b Bump google.golang.org/grpc from 1.64.0 to 1.64.1 in /sdks 
(#31817)

No new revisions were added by this update.

Summary of changes:
 sdks/go.mod | 2 +-
 sdks/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)



(beam) branch master updated: Bump certifi from 2024.2.2 to 2024.7.4 in /sdks/python/container/py38 (#31790)

2024-07-09 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new a4558dfd8c8 Bump certifi from 2024.2.2 to 2024.7.4 in 
/sdks/python/container/py38 (#31790)
a4558dfd8c8 is described below

commit a4558dfd8c8ac46c32d5d121a66e24e7e36381b2
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
AuthorDate: Tue Jul 9 16:30:44 2024 -0400

Bump certifi from 2024.2.2 to 2024.7.4 in /sdks/python/container/py38 
(#31790)

Bumps [certifi](https://github.com/certifi/python-certifi) from 2024.2.2 to 
2024.7.4.
- 
[Commits](https://github.com/certifi/python-certifi/compare/2024.02.02...2024.07.04)

---
updated-dependencies:
- dependency-name: certifi
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] 
<49699333+dependabot[bot]@users.noreply.github.com>
---
 sdks/python/container/py38/base_image_requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/python/container/py38/base_image_requirements.txt 
b/sdks/python/container/py38/base_image_requirements.txt
index 9cfd30ba6b3..f88dba10346 100644
--- a/sdks/python/container/py38/base_image_requirements.txt
+++ b/sdks/python/container/py38/base_image_requirements.txt
@@ -29,7 +29,7 @@ beautifulsoup4==4.12.3
 bs4==0.0.2
 build==1.2.1
 cachetools==5.3.3
-certifi==2024.2.2
+certifi==2024.7.4
 cffi==1.16.0
 charset-normalizer==3.3.2
 click==8.1.7



(beam) branch master updated: Add support for WindowStrategy Pane and AllowedLatness features (#31806)

2024-07-09 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 516bbc77ef3 Add support for WindowStrategy Pane and AllowedLatness 
features (#31806)
516bbc77ef3 is described below

commit 516bbc77ef3886379983282ea53713bffba5290c
Author: Damon 
AuthorDate: Tue Jul 9 08:50:49 2024 -0700

Add support for WindowStrategy Pane and AllowedLatness features (#31806)
---
 sdks/go/pkg/beam/runners/prism/internal/handlerunner.go   | 4 
 sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go 
b/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
index a205c768731..910fa0a0ca8 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
@@ -297,6 +297,10 @@ func gbkBytes(ws *pipepb.WindowingStrategy, wc, kc, vc 
*pipepb.Coder, toAggregat
outputTime = func(w typex.Window, et mtime.Time) mtime.Time {
return w.MaxTimestamp()
}
+   case pipepb.OutputTime_EARLIEST_IN_PANE, 
pipepb.OutputTime_LATEST_IN_PANE:
+   outputTime = func(w typex.Window, et mtime.Time) mtime.Time {
+   return et
+   }
default:
// TODO need to correct session logic if output time is 
different.
panic(fmt.Sprintf("unsupported OutputTime behavior: %v", 
ws.GetOutputTime()))
diff --git a/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go 
b/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
index 737a1b22276..3efe48e2311 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
@@ -22,6 +22,7 @@ import (
"sync"
"sync/atomic"
 
+   "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime"
jobpb 
"github.com/apache/beam/sdks/v2/go/pkg/beam/model/jobmanagement_v1"
pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1"
"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/urns"
@@ -195,7 +196,8 @@ func (s *Server) Prepare(ctx context.Context, req 
*jobpb.PrepareJobRequest) (*jo
 
// Inspect Windowing strategies for unsupported features.
for wsID, ws := range 
job.Pipeline.GetComponents().GetWindowingStrategies() {
-   check("WindowingStrategy.AllowedLateness", 
ws.GetAllowedLateness(), int64(0))
+   check("WindowingStrategy.AllowedLateness", 
ws.GetAllowedLateness(), int64(0), mtime.MaxTimestamp.Milliseconds())
+
// Both Closing behaviors are identical without additional 
trigger firings.
check("WindowingStrategy.ClosingBehaviour", 
ws.GetClosingBehavior(), pipepb.ClosingBehavior_EMIT_IF_NONEMPTY, 
pipepb.ClosingBehavior_EMIT_ALWAYS)
check("WindowingStrategy.AccumulationMode", 
ws.GetAccumulationMode(), pipepb.AccumulationMode_DISCARDING)



(beam) branch master updated (8a88f1583f0 -> 746f3c5557e)

2024-07-08 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


from 8a88f1583f0 Solace Read connector: adding implementations of 
SempClient and SempClientFactory (#31542)
 add 746f3c5557e Use go 1.22 for self-hosted GHAs (#31767)

No new revisions were added by this update.

Summary of changes:
 .github/actions/setup-environment-action/action.yml |  6 --
 .github/workflows/go_tests.yml  |  7 +++
 .github/workflows/local_env_tests.yml   | 18 --
 3 files changed, 15 insertions(+), 16 deletions(-)



(beam) branch master updated: Bump cloud.google.com/go/storage from 1.41.0 to 1.43.0 in /sdks (#31772)

2024-07-08 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new d1df1d7ecc9 Bump cloud.google.com/go/storage from 1.41.0 to 1.43.0 in 
/sdks (#31772)
d1df1d7ecc9 is described below

commit d1df1d7ecc9f2b123586d07311f61cd3c34a2f77
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
AuthorDate: Mon Jul 8 09:59:26 2024 -0400

Bump cloud.google.com/go/storage from 1.41.0 to 1.43.0 in /sdks (#31772)

Bumps 
[cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) 
from 1.41.0 to 1.43.0.
- [Release notes](https://github.com/googleapis/google-cloud-go/releases)
- 
[Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md)
- 
[Commits](https://github.com/googleapis/google-cloud-go/compare/spanner/v1.41.0...spanner/v1.43.0)

---
updated-dependencies:
- dependency-name: cloud.google.com/go/storage
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] 
<49699333+dependabot[bot]@users.noreply.github.com>
---
 sdks/go.mod | 16 
 sdks/go.sum | 34 +-
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/sdks/go.mod b/sdks/go.mod
index b156c8beac2..b275e47f5c1 100644
--- a/sdks/go.mod
+++ b/sdks/go.mod
@@ -27,9 +27,9 @@ require (
cloud.google.com/go/bigtable v1.25.0
cloud.google.com/go/datastore v1.17.1
cloud.google.com/go/profiler v0.4.0
-   cloud.google.com/go/pubsub v1.38.0
+   cloud.google.com/go/pubsub v1.39.0
cloud.google.com/go/spanner v1.63.0
-   cloud.google.com/go/storage v1.41.0
+   cloud.google.com/go/storage v1.43.0
github.com/aws/aws-sdk-go-v2 v1.30.0
github.com/aws/aws-sdk-go-v2/config v1.27.4
github.com/aws/aws-sdk-go-v2/credentials v1.17.18
@@ -59,8 +59,8 @@ require (
golang.org/x/sync v0.7.0
golang.org/x/sys v0.21.0
golang.org/x/text v0.16.0
-   google.golang.org/api v0.184.0
-   google.golang.org/genproto v0.0.0-20240604185151-ef581f913117
+   google.golang.org/api v0.187.0
+   google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d
google.golang.org/grpc v1.64.0
google.golang.org/protobuf v1.34.2
gopkg.in/yaml.v2 v2.4.0
@@ -74,7 +74,7 @@ require (
 )
 
 require (
-   cloud.google.com/go/auth v0.5.1 // indirect
+   cloud.google.com/go/auth v0.6.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
dario.cat/mergo v1.0.0 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
@@ -151,7 +151,7 @@ require (
github.com/google/renameio/v2 v2.0.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
-   github.com/googleapis/gax-go/v2 v2.12.4 // indirect
+   github.com/googleapis/gax-go/v2 v2.12.5 // indirect
github.com/gorilla/handlers v1.5.2 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -183,6 +183,6 @@ require (
golang.org/x/mod v0.17.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
-   google.golang.org/genproto/googleapis/api 
v0.0.0-20240604185151-ef581f913117 // indirect
-   google.golang.org/genproto/googleapis/rpc 
v0.0.0-20240610135401-a8a62080eff3 // indirect
+   google.golang.org/genproto/googleapis/api 
v0.0.0-20240617180043-68d350f18fd4 // indirect
+   google.golang.org/genproto/googleapis/rpc 
v0.0.0-20240624140628-dc46fd24d27d // indirect
 )
diff --git a/sdks/go.sum b/sdks/go.sum
index 2b2862bb2f0..4b8f7f0d458 100644
--- a/sdks/go.sum
+++ b/sdks/go.sum
@@ -99,8 +99,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod 
h1:z/736/oNmtGAyU47reJgGN+KVo
 cloud.google.com/go/assuredworkloads v1.8.0/go.mod 
h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
 cloud.google.com/go/assuredworkloads v1.9.0/go.mod 
h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
 cloud.google.com/go/assuredworkloads v1.10.0/go.mod 
h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
-cloud.google.com/go/auth v0.5.1 h1:0QNO7VThG54LUzKiQxv8C6x1YX7lUrzlAa1nVLF8CIw=
-cloud.google.com/go/auth v0.5.1/go.mod 
h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s=
+cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38=
+cloud.google.com/go/auth v0.6.1/go.mod 
h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4=
 cloud.google.com/go/auth/oauth2adapt v0.2.2 
h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4=
 cloud.google.com/go/auth

(beam) branch master updated: Fix installation invocation for OpenJDK (#31752)

2024-07-02 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new e28e00ac608 Fix installation invocation for OpenJDK (#31752)
e28e00ac608 is described below

commit e28e00ac60840535fd60596237223faf10c84c77
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Tue Jul 2 15:35:36 2024 -0400

Fix installation invocation for OpenJDK (#31752)
---
 playground/backend/containers/python/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/playground/backend/containers/python/Dockerfile 
b/playground/backend/containers/python/Dockerfile
index b617e02b15c..ca57bef9025 100644
--- a/playground/backend/containers/python/Dockerfile
+++ b/playground/backend/containers/python/Dockerfile
@@ -66,7 +66,7 @@ RUN cd /opt/playground/backend/kafka-emulator/ && tar -xvf 
kafka-emulator.tar &&
 mv beam-playground-kafka-emulator-*.jar beam-playground-kafka-emulator.jar
 RUN apt-get update && \
 wget 
http://http.us.debian.org/debian/pool/main/o/openjdk-11/openjdk-11-jre-headless_11.0.23+9-1~deb11u1_amd64.deb
 && \
-apt install -y ./openjdk-11-jre-headless_11.0.22+7-1~deb11u1_amd64.deb
+apt install -y ./openjdk-11-jre-headless_11.0.23+9-1~deb11u1_amd64.deb
 
 # Create a user group `appgroup` and a user `appuser`
 RUN groupadd --gid 2 appgroup \



(beam) branch master updated: Update Playground Python Dockerfile (#31745)

2024-07-02 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new b5035a08836 Update Playground Python Dockerfile (#31745)
b5035a08836 is described below

commit b5035a08836c31451c5015c94f2a05fbb107febd
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Tue Jul 2 15:12:19 2024 -0400

Update Playground Python Dockerfile (#31745)

* Fix Python Playground Dockerfile

* Update version to 11.0.23
---
 playground/backend/containers/python/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/playground/backend/containers/python/Dockerfile 
b/playground/backend/containers/python/Dockerfile
index 709a22eec96..b617e02b15c 100644
--- a/playground/backend/containers/python/Dockerfile
+++ b/playground/backend/containers/python/Dockerfile
@@ -65,7 +65,7 @@ RUN cd /opt/playground/backend/kafka-emulator/ && tar -xvf 
kafka-emulator.tar &&
 mv kafka-emulator/*.jar . && rmdir kafka-emulator/ &&\
 mv beam-playground-kafka-emulator-*.jar beam-playground-kafka-emulator.jar
 RUN apt-get update && \
-wget 
http://http.us.debian.org/debian/pool/main/o/openjdk-11/openjdk-11-jre-headless_11.0.22+7-1~deb11u1_amd64.deb
 && \
+wget 
http://http.us.debian.org/debian/pool/main/o/openjdk-11/openjdk-11-jre-headless_11.0.23+9-1~deb11u1_amd64.deb
 && \
 apt install -y ./openjdk-11-jre-headless_11.0.22+7-1~deb11u1_amd64.deb
 
 # Create a user group `appgroup` and a user `appuser`



(beam) branch master updated: Update Staticcheck to 2023.1.7 (#31750)

2024-07-02 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new dc63b0ffc99 Update Staticcheck to 2023.1.7 (#31750)
dc63b0ffc99 is described below

commit dc63b0ffc99b06e72bbfbb25c54f1cb8dbeb4968
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Tue Jul 2 13:50:08 2024 -0400

Update Staticcheck to 2023.1.7 (#31750)
---
 .github/workflows/go_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/go_tests.yml b/.github/workflows/go_tests.yml
index c7eed447095..db30bac68ec 100644
--- a/.github/workflows/go_tests.yml
+++ b/.github/workflows/go_tests.yml
@@ -65,6 +65,6 @@ jobs:
   go vet --copylocks=false --unsafeptr=false ./...
   - name: Run Staticcheck
 run: |
-  go install "honnef.co/go/tools/cmd/staticcheck@2023.1.3"
+  go install "honnef.co/go/tools/cmd/staticcheck@2023.1.7"
   cd sdks/go/pkg/beam
   $(go env GOPATH)/bin/staticcheck ./...



(beam) branch master updated (45733be4fe4 -> 8e11d0a4815)

2024-07-02 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


from 45733be4fe4 Do not build shadowjar and disable javadoc for it modules 
(#31741)
 add 8e11d0a4815 Update Go Version to 1.22.4 (#31747)

No new revisions were added by this update.

Summary of changes:
 .github/actions/setup-default-test-properties/test-properties.json| 2 +-
 .github/gh-actions-self-hosted-runners/arc/images/Dockerfile  | 2 +-
 .github/workflows/go_tests.yml| 2 +-
 .github/workflows/local_env_tests.yml | 4 ++--
 .../src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy| 2 +-
 dev-support/docker/Dockerfile | 2 +-
 sdks/go/run_with_go_version.sh| 2 +-
 7 files changed, 8 insertions(+), 8 deletions(-)



(beam) branch master updated: Add a unit test for MakePipelineOptionsFileAndEnvVar (#31732)

2024-07-01 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new bae6fcfcb49 Add a unit test for MakePipelineOptionsFileAndEnvVar 
(#31732)
bae6fcfcb49 is described below

commit bae6fcfcb492db64e8b7333d928fdbecdcb3e21f
Author: ljjulia 
AuthorDate: Mon Jul 1 06:20:33 2024 -0700

Add a unit test for MakePipelineOptionsFileAndEnvVar (#31732)

* Add unit test for MakePipelineOptionsFileAndEnvVar

* Add coverage for validating options string is JSON
---
 sdks/go/container/tools/pipeline_options.go|  5 +++
 ...ipeline_options.go => pipeline_options_test.go} | 49 +++---
 2 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/sdks/go/container/tools/pipeline_options.go 
b/sdks/go/container/tools/pipeline_options.go
index 7b46d8fa8c8..026fb31b099 100644
--- a/sdks/go/container/tools/pipeline_options.go
+++ b/sdks/go/container/tools/pipeline_options.go
@@ -16,6 +16,7 @@
 package tools
 
 import (
+   "encoding/json"
"fmt"
"os"
 )
@@ -31,6 +32,10 @@ func MakePipelineOptionsFileAndEnvVar(options string) error {
return fmt.Errorf("unable to create %v: %w", fn, err)
}
defer f.Close()
+   var js map[string]interface{}
+   if json.Unmarshal([]byte(options), ) != nil {
+   return fmt.Errorf("options string is not JSON formatted %v", 
options)
+   }
if _, err := f.WriteString(options); err != nil {
return fmt.Errorf("error writing %v: %w", f.Name(), err)
}
diff --git a/sdks/go/container/tools/pipeline_options.go 
b/sdks/go/container/tools/pipeline_options_test.go
similarity index 54%
copy from sdks/go/container/tools/pipeline_options.go
copy to sdks/go/container/tools/pipeline_options_test.go
index 7b46d8fa8c8..7a0d7ebd5f0 100644
--- a/sdks/go/container/tools/pipeline_options.go
+++ b/sdks/go/container/tools/pipeline_options_test.go
@@ -16,24 +16,43 @@
 package tools
 
 import (
-   "fmt"
"os"
+   "testing"
 )
 
-// MakePipelineOptionsFileAndEnvVar writes the pipeline options to a file.
-// Assumes the options string is JSON formatted.
-//
-// Stores the file name in question in PIPELINE_OPTIONS_FILE for access by the 
SDK.
-func MakePipelineOptionsFileAndEnvVar(options string) error {
-   fn := "pipeline_options.json"
-   f, err := os.Create(fn)
-   if err != nil {
-   return fmt.Errorf("unable to create %v: %w", fn, err)
+func TestMakePipelineOptionsFileAndEnvVar(t *testing.T) {
+   tests := []struct {
+   name  string
+   inputOptions  string
+   expectedError string
+   }{
+   {
+   "empty options",
+   "{}",
+   "",
+   },
+   {
+   "valid options",
+   "{\"abc\": 123}",
+   "",
+   },
+   {
+   "invalid options",
+   "{4}",
+   "options string is not JSON formatted {4}",
+   },
}
-   defer f.Close()
-   if _, err := f.WriteString(options); err != nil {
-   return fmt.Errorf("error writing %v: %w", f.Name(), err)
+
+   for _, test := range tests {
+   t.Run(test.name, func(t *testing.T) {
+   t.Cleanup(os.Clearenv)
+   err := 
MakePipelineOptionsFileAndEnvVar(test.inputOptions)
+   if err != nil {
+   if got, want := err.Error(), 
test.expectedError; got != want {
+   t.Errorf("got error: %v, want error: 
%v", got, want)
+   }
+   }
+   })
}
-   os.Setenv("PIPELINE_OPTIONS_FILE", f.Name())
-   return nil
+   os.Remove("pipeline_options.json")
 }



(beam-starter-go) 01/01: Merge pull request #38 from apache/dependabot/go_modules/github.com/apache/beam/sdks/v2-2.57.0

2024-06-27 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/beam-starter-go.git

commit 0f5af0e96cad628fe68b2b6de7763d01a5670c6f
Merge: ccefb68 f375a73
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Thu Jun 27 11:19:14 2024 -0400

Merge pull request #38 from 
apache/dependabot/go_modules/github.com/apache/beam/sdks/v2-2.57.0

Bump github.com/apache/beam/sdks/v2 from 2.56.0 to 2.57.0

 go.mod | 10 +-
 go.sum | 20 ++--
 2 files changed, 15 insertions(+), 15 deletions(-)



(beam-starter-go) branch main updated (ccefb68 -> 0f5af0e)

2024-06-27 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/beam-starter-go.git


from ccefb68  Bump github.com/apache/beam/sdks/v2 from 2.55.1 to 2.56.0 
(#37)
 add f375a73  Bump github.com/apache/beam/sdks/v2 from 2.56.0 to 2.57.0
 new 0f5af0e  Merge pull request #38 from 
apache/dependabot/go_modules/github.com/apache/beam/sdks/v2-2.57.0

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 go.mod | 10 +-
 go.sum | 20 ++--
 2 files changed, 15 insertions(+), 15 deletions(-)



(beam) branch master updated: Bump github.com/go-sql-driver/mysql from 1.8.0 to 1.8.1 in /sdks (#31688)

2024-06-26 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 4a5eaf21961 Bump github.com/go-sql-driver/mysql from 1.8.0 to 1.8.1 in 
/sdks (#31688)
4a5eaf21961 is described below

commit 4a5eaf21961228188c8add70148a477b50af10a8
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
AuthorDate: Wed Jun 26 15:36:24 2024 -0400

Bump github.com/go-sql-driver/mysql from 1.8.0 to 1.8.1 in /sdks (#31688)

Bumps 
[github.com/go-sql-driver/mysql](https://github.com/go-sql-driver/mysql) from 
1.8.0 to 1.8.1.
- [Release notes](https://github.com/go-sql-driver/mysql/releases)
- 
[Changelog](https://github.com/go-sql-driver/mysql/blob/master/CHANGELOG.md)
- [Commits](https://github.com/go-sql-driver/mysql/compare/v1.8.0...v1.8.1)

---
updated-dependencies:
- dependency-name: github.com/go-sql-driver/mysql
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] 
<49699333+dependabot[bot]@users.noreply.github.com>
---
 sdks/go.mod | 2 +-
 sdks/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/sdks/go.mod b/sdks/go.mod
index 1109ee80086..e7443172de0 100644
--- a/sdks/go.mod
+++ b/sdks/go.mod
@@ -38,7 +38,7 @@ require (
github.com/aws/smithy-go v1.20.2
github.com/docker/go-connections v0.5.0
github.com/dustin/go-humanize v1.0.1
-   github.com/go-sql-driver/mysql v1.8.0
+   github.com/go-sql-driver/mysql v1.8.1
github.com/golang/protobuf v1.5.4 // TODO(danoliveira): Fully replace 
this with google.golang.org/protobuf
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
diff --git a/sdks/go.sum b/sdks/go.sum
index 20234f53dd5..19fd97db5e7 100644
--- a/sdks/go.sum
+++ b/sdks/go.sum
@@ -827,8 +827,8 @@ github.com/go-ole/go-ole v1.2.6/go.mod 
h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU
 github.com/go-pdf/fpdf v0.5.0/go.mod 
h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
 github.com/go-pdf/fpdf v0.6.0/go.mod 
h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
 github.com/go-sql-driver/mysql v1.5.0/go.mod 
h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
-github.com/go-sql-driver/mysql v1.8.0 
h1:UtktXaU2Nb64z/pLiGIxY4431SJ4/dR5cjMmlVHgnT4=
-github.com/go-sql-driver/mysql v1.8.0/go.mod 
h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/go-sql-driver/mysql v1.8.1 
h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod 
h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/goccy/go-json v0.9.11/go.mod 
h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.10.2 
h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod 
h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=



(beam) branch master updated (0f2e1963987 -> a944bf87cd0)

2024-06-24 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


from 0f2e1963987 Replace getElementType with getValueType for MAP in 
AvroGenericRecord… (#31653)
 add a944bf87cd0 Bump github.com/aws/aws-sdk-go-v2 from 1.27.2 to 1.30.0 in 
/sdks (#31670)

No new revisions were added by this update.

Summary of changes:
 sdks/go.mod | 2 +-
 sdks/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)



(beam) branch master updated: Bump golang.org/x/net from 0.24.0 to 0.26.0 in /sdks (#31614)

2024-06-21 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 3695d496701 Bump golang.org/x/net from 0.24.0 to 0.26.0 in /sdks 
(#31614)
3695d496701 is described below

commit 3695d496701cf65fd296ac2b046746aa48bb6526
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
AuthorDate: Fri Jun 21 10:07:54 2024 -0400

Bump golang.org/x/net from 0.24.0 to 0.26.0 in /sdks (#31614)

Bumps [golang.org/x/net](https://github.com/golang/net) from 0.24.0 to 
0.26.0.
- [Commits](https://github.com/golang/net/compare/v0.24.0...v0.26.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] 
<49699333+dependabot[bot]@users.noreply.github.com>
---
 sdks/go.mod | 14 +++---
 sdks/go.sum | 28 ++--
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/sdks/go.mod b/sdks/go.mod
index 8ced6fbe1f9..ed1d980928c 100644
--- a/sdks/go.mod
+++ b/sdks/go.mod
@@ -54,11 +54,11 @@ require (
github.com/xitongsys/parquet-go v1.6.2
github.com/xitongsys/parquet-go-source 
v0.0.0-20220315005136-aec0fe3e777c
go.mongodb.org/mongo-driver v1.13.1
-   golang.org/x/net v0.24.0
+   golang.org/x/net v0.26.0
golang.org/x/oauth2 v0.18.0
-   golang.org/x/sync v0.6.0
-   golang.org/x/sys v0.19.0
-   golang.org/x/text v0.14.0
+   golang.org/x/sync v0.7.0
+   golang.org/x/sys v0.21.0
+   golang.org/x/text v0.16.0
google.golang.org/api v0.171.0
google.golang.org/genproto v0.0.0-20240308144416-29370a3891b7
google.golang.org/grpc v1.63.2
@@ -181,9 +181,9 @@ require (
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opencensus.io v0.24.0 // indirect
-   golang.org/x/crypto v0.22.0 // indirect
-   golang.org/x/mod v0.13.0 // indirect
-   golang.org/x/tools v0.14.0 // indirect
+   golang.org/x/crypto v0.24.0 // indirect
+   golang.org/x/mod v0.17.0 // indirect
+   golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/api 
v0.0.0-20240314234333-6e1732d8331c // indirect
diff --git a/sdks/go.sum b/sdks/go.sum
index 20b520b5520..56ee80f8faa 100644
--- a/sdks/go.sum
+++ b/sdks/go.sum
@@ -494,8 +494,8 @@ golang.org/x/crypto 
v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod 
h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod 
h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod 
h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.22.0/go.mod 
h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod 
h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod 
h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod 
h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod 
h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -528,8 +528,8 @@ golang.org/x/mod 
v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod 
h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
-golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod 
h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod 
h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod 
h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -553,8 +553,8 @@ golang.org/x/net v0.0.0-20201110031124-

(beam) branch master updated: Bump github.com/linkedin/goavro/v2 from 2.12.0 to 2.13.0 in /sdks (#31633)

2024-06-21 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 999b70b0c48 Bump github.com/linkedin/goavro/v2 from 2.12.0 to 2.13.0 
in /sdks (#31633)
999b70b0c48 is described below

commit 999b70b0c48dc76a73e9aa1006b4f0b9eda552d7
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
AuthorDate: Fri Jun 21 10:07:03 2024 -0400

Bump github.com/linkedin/goavro/v2 from 2.12.0 to 2.13.0 in /sdks (#31633)

Bumps [github.com/linkedin/goavro/v2](https://github.com/linkedin/goavro) 
from 2.12.0 to 2.13.0.
- [Release notes](https://github.com/linkedin/goavro/releases)
- 
[Changelog](https://github.com/linkedin/goavro/blob/master/debug_release.go)
- [Commits](https://github.com/linkedin/goavro/compare/v2.12...v2.13.0)

---
updated-dependencies:
- dependency-name: github.com/linkedin/goavro/v2
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] 
<49699333+dependabot[bot]@users.noreply.github.com>
---
 sdks/go.mod | 2 +-
 sdks/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/sdks/go.mod b/sdks/go.mod
index ec96f146317..8ced6fbe1f9 100644
--- a/sdks/go.mod
+++ b/sdks/go.mod
@@ -44,7 +44,7 @@ require (
github.com/google/uuid v1.6.0
github.com/johannesboyne/gofakes3 v0.0.0-20221110173912-32fb85c5aed6
github.com/lib/pq v1.10.9
-   github.com/linkedin/goavro/v2 v2.12.0
+   github.com/linkedin/goavro/v2 v2.13.0
github.com/nats-io/nats-server/v2 v2.10.12
github.com/nats-io/nats.go v1.33.1
github.com/proullon/ramsql v0.1.3
diff --git a/sdks/go.sum b/sdks/go.sum
index bb0d3286d8e..20b520b5520 100644
--- a/sdks/go.sum
+++ b/sdks/go.sum
@@ -327,8 +327,8 @@ github.com/kr/text v0.2.0 
h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod 
h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod 
h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/linkedin/goavro/v2 v2.12.0 
h1:rIQQSj8jdAUlKQh6DttK8wCRv4t4QO09g1C4aBWXslg=
-github.com/linkedin/goavro/v2 v2.12.0/go.mod 
h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
+github.com/linkedin/goavro/v2 v2.13.0 
h1:L8eI8GcuciwUkt41Ej62joSZS4kKaYIUdze+6for9NU=
+github.com/linkedin/goavro/v2 v2.13.0/go.mod 
h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 
h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod 
h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/magiconair/properties v1.8.7 
h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=



(beam) branch master updated: Update documentation for new TFT Transforms (#31565)

2024-06-12 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 9c80b81fd47 Update documentation for new TFT Transforms (#31565)
9c80b81fd47 is described below

commit 9c80b81fd47a2b8f13db65cbc9dde8cf007af5e8
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Wed Jun 12 12:04:20 2024 -0400

Update documentation for new TFT Transforms (#31565)

* Update documentation for new TFT Transforms

* narrow versioning scope
---
 sdks/python/apache_beam/ml/transforms/tft.py| 4 
 website/www/site/content/en/documentation/ml/preprocess-data.md | 7 ++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/ml/transforms/tft.py 
b/sdks/python/apache_beam/ml/transforms/tft.py
index e03d52214c9..6903cca8941 100644
--- a/sdks/python/apache_beam/ml/transforms/tft.py
+++ b/sdks/python/apache_beam/ml/transforms/tft.py
@@ -52,13 +52,17 @@ __all__ = [
 'ComputeAndApplyVocabulary',
 'ScaleToZScore',
 'ScaleTo01',
+'ScaleToGaussian',
 'ApplyBuckets',
+'ApplyBucketsWithInterpolation',
 'Bucketize',
 'TFIDF',
 'TFTOperation',
 'ScaleByMinMax',
 'NGrams',
 'BagOfWords',
+'HashStrings',
+'DeduplicateTensorPerRow',
 ]
 
 # Register the expected input types for each operation
diff --git a/website/www/site/content/en/documentation/ml/preprocess-data.md 
b/website/www/site/content/en/documentation/ml/preprocess-data.md
index 1365926d3cc..fb21cc8928b 100644
--- a/website/www/site/content/en/documentation/ml/preprocess-data.md
+++ b/website/www/site/content/en/documentation/ml/preprocess-data.md
@@ -43,7 +43,7 @@ available transforms, see the [Transforms](#transforms) 
section on this page.
 ## Support and limitations {#support}
 
 -   Available in the Apache Beam Python SDK versions 2.53.0 and later.
--   Supports Python 3.8, 3.9, and 3.10.
+-   Supports Python 3.8, 3.9, 3.10, and 3.11
 -   Only available for pipelines that use [default 
windows](/documentation/programming-guide/#single-global-window).
 
 ## Transforms {#transforms}
@@ -74,11 +74,16 @@ TensorFlow documentation.
 | Transform name | Description |
 | --- | ---|
 | ApplyBuckets | See 
[`tft.apply_buckets`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/apply_buckets)
 in the TensorFlow documentation. |
+| ApplyBucketsWithInterpolation | See 
[`tft.apply_buckets_with_interpolation`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/apply_buckets_with_interpolation)
 in the TensorFlow documentation. |
+| BagOfWords | See 
[`tft.bag_of_words`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/bag_of_words)
 in the TensorFlow documentation. |
 | Bucketize | See 
[`tft.bucketize`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/bucketize)
 in the TensorFlow documentation. |
 | ComputeAndApplyVocabulary | See 
[`tft.compute_and_apply_vocabulary`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/compute_and_apply_vocabulary)
 in the TensorFlow documentation. |
+| DeduplicateTensorPerRow | See 
[`tft.deduplicate_tensor_per_row`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/deduplicate_tensor_per_row)
 in the TensorFlow documentation. |
+| HashStrings | See 
[`tft.hash_strings`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/hash_strings)
 in the TensorFlow documentation. |
 | NGrams | See 
[`tft.ngrams`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/ngrams)
 in the TensorFlow documentation. |
 | ScaleByMinMax | See 
[`tft.scale_by_min_max`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_by_min_max)
 in the TensorFlow documentation. |
 | ScaleTo01 | See 
[`tft.scale_to_0_1`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_to_0_1)
 in the TensorFlow documentation. |
+| ScaleToGaussian | See 
[`tft.scale_to_gaussian`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_to_gaussian)
 in the TensorFlow documentation. |
 | ScaleToZScore | See 
[`tft.scale_to_z_score`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_to_z_score)
 in the TensorFlow documentation. |
 | TFIDF | See 
[`tft.tfidf`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/tfidf)
 in the TensorFlow documentation. |:
 {{< /table >}}



(beam) branch master updated: Add image embedding support to TFHub MLTransforms (#31564)

2024-06-12 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 7732e11acb7 Add image embedding support to TFHub MLTransforms (#31564)
7732e11acb7 is described below

commit 7732e11acb7e215649748f0cff32212a4c0d777d
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Wed Jun 12 12:03:45 2024 -0400

Add image embedding support to TFHub MLTransforms (#31564)

* Add image embedding support to TFHub MLTransforms

* linting

* more linting

* formatting

* typo
---
 .../ml/transforms/embeddings/tensorflow_hub.py | 42 ++-
 .../transforms/embeddings/tensorflow_hub_test.py   | 60 ++
 2 files changed, 101 insertions(+), 1 deletion(-)

diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py 
b/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py
index 9e448078825..f78ddf3ff04 100644
--- a/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py
+++ b/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py
@@ -29,9 +29,10 @@ from apache_beam.ml.inference.base import RunInference
 from apache_beam.ml.inference.tensorflow_inference import TFModelHandlerTensor
 from apache_beam.ml.inference.tensorflow_inference import 
default_tensor_inference_fn
 from apache_beam.ml.transforms.base import EmbeddingsManager
+from apache_beam.ml.transforms.base import _ImageEmbeddingHandler
 from apache_beam.ml.transforms.base import _TextEmbeddingHandler
 
-__all__ = ['TensorflowHubTextEmbeddings']
+__all__ = ['TensorflowHubTextEmbeddings', 'TensorflowHubImageEmbeddings']
 
 
 # TODO: https://github.com/apache/beam/issues/30288
@@ -132,3 +133,42 @@ class TensorflowHubTextEmbeddings(EmbeddingsManager):
 model_handler=_TextEmbeddingHandler(self),
 inference_args=self.inference_args,
 ))
+
+
+class TensorflowHubImageEmbeddings(EmbeddingsManager):
+  def __init__(self, columns: List[str], hub_url: str, **kwargs):
+"""
+Embedding config for tensorflow hub models. This config can be used with
+MLTransform to embed image data. Models are loaded using the RunInference
+PTransform with the help of a ModelHandler.
+
+Args:
+  columns: The columns containing the images to be embedded.
+  hub_url: The url of the tensorflow hub model.
+  min_batch_size: The minimum batch size to be used for inference.
+  max_batch_size: The maximum batch size to be used for inference.
+  large_model: Whether to share the model across processes.
+"""
+super().__init__(columns=columns, **kwargs)
+self.model_uri = hub_url
+
+  def get_model_handler(self) -> ModelHandler:
+# override the default inference function
+return _TensorflowHubModelHandler(
+model_uri=self.model_uri,
+preprocessing_url=None,
+min_batch_size=self.min_batch_size,
+max_batch_size=self.max_batch_size,
+large_model=self.large_model,
+)
+
+  def get_ptransform_for_processing(self, **kwargs) -> beam.PTransform:
+"""
+Returns a RunInference object that is used to run inference on the text 
+input using _ImageEmbeddingHandler.
+"""
+return (
+RunInference(
+model_handler=_ImageEmbeddingHandler(self),
+inference_args=self.inference_args,
+))
diff --git 
a/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py 
b/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py
index b08ca8e2d8e..24bca5155fa 100644
--- a/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py
+++ b/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py
@@ -20,10 +20,13 @@ import tempfile
 import unittest
 import uuid
 
+import numpy as np
+
 import apache_beam as beam
 from apache_beam.ml.transforms.base import MLTransform
 
 hub_url = 'https://tfhub.dev/google/nnlm-en-dim128/2'
+hub_img_url = 
'https://www.kaggle.com/models/google/resnet-v2/TensorFlow2/101-feature-vector/2'
  # pylint: disable=line-too-long
 test_query_column = 'test_query'
 test_query = 'This is a test query'
 
@@ -32,6 +35,7 @@ try:
   from apache_beam.ml.transforms.embeddings.tensorflow_hub import 
TensorflowHubTextEmbeddings
 except ImportError:
   TensorflowHubTextEmbeddings = None  # type: ignore
+  tf = None
 
 # pylint: disable=ungrouped-imports
 try:
@@ -40,6 +44,14 @@ try:
 except ImportError:
   tft = None
 
+# pylint: disable=ungrouped-imports
+try:
+  from apache_beam.ml.transforms.embeddings.tensorflow_hub import 
TensorflowHubImageEmbeddings
+  from PIL import Image
+except ImportError:
+  TensorflowHubImageEmbeddings = None  # type: ignore
+  Image = None
+
 

(beam) branch master updated: Update Python version in the dev environment script. (#31416)

2024-06-11 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 6a24b07b6c6 Update Python version in the dev environment script. 
(#31416)
6a24b07b6c6 is described below

commit 6a24b07b6c69f4079202f8740a1e42da8494506c
Author: tvalentyn 
AuthorDate: Tue Jun 11 08:24:15 2024 -0700

Update Python version in the dev environment script. (#31416)

* Update Python version in the dev environment script.

* fixup
---
 local-env-setup.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/local-env-setup.sh b/local-env-setup.sh
index 6cd1092023a..f13dc88432a 100755
--- a/local-env-setup.sh
+++ b/local-env-setup.sh
@@ -55,7 +55,7 @@ if [ "$kernelname" = "Linux" ]; then
 exit
 fi
 
-for ver in 3.8 3.9 3.10 3.11 3; do
+for ver in 3.8 3.9 3.10 3.11 3.12 3; do
 apt install --yes python$ver-venv
 done
 
@@ -89,7 +89,7 @@ elif [ "$kernelname" = "Darwin" ]; then
 echo "Installing openjdk@8"
 brew install openjdk@8
 fi
-for ver in 3.8 3.9 3.10 3.11; do
+for ver in 3.8 3.9 3.10 3.11 3.12; do
   if brew ls --versions python@$ver > /dev/null; then
   echo "python@$ver already installed. Skipping"
   brew info python@$ver



(beam) branch master updated: Implement Hugging Face Image Embedding MLTransform (#31536)

2024-06-07 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new fe54c211512 Implement Hugging Face Image Embedding MLTransform (#31536)
fe54c211512 is described below

commit fe54c211512ae90f0d596bf2ed1e40d3826a7c7e
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Fri Jun 7 11:21:02 2024 -0400

Implement Hugging Face Image Embedding MLTransform (#31536)

* Implement Hugging Face Image Embedding MLTransform

* correct imports

* Simplify to original sentence transformer class
---
 .../ml/transforms/embeddings/huggingface.py| 19 ++--
 .../ml/transforms/embeddings/huggingface_test.py   | 52 +-
 2 files changed, 67 insertions(+), 4 deletions(-)

diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/huggingface.py 
b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface.py
index 7fcaa9c9a5d..46b4ef9cf7d 100644
--- a/sdks/python/apache_beam/ml/transforms/embeddings/huggingface.py
+++ b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface.py
@@ -32,6 +32,7 @@ import apache_beam as beam
 from apache_beam.ml.inference.base import ModelHandler
 from apache_beam.ml.inference.base import RunInference
 from apache_beam.ml.transforms.base import EmbeddingsManager
+from apache_beam.ml.transforms.base import _ImageEmbeddingHandler
 from apache_beam.ml.transforms.base import _TextEmbeddingHandler
 
 try:
@@ -114,6 +115,7 @@ class SentenceTransformerEmbeddings(EmbeddingsManager):
   model_name: str,
   columns: List[str],
   max_seq_length: Optional[int] = None,
+  image_model: bool = False,
   **kwargs):
 """
 Embedding config for sentence-transformers. This config can be used with
@@ -122,9 +124,13 @@ class SentenceTransformerEmbeddings(EmbeddingsManager):
 
 Args:
   model_name: Name of the model to use. The model should be hosted on
-HuggingFace Hub or compatible with sentence_transformers.
+HuggingFace Hub or compatible with sentence_transformers. For image
+embedding models, see
+
https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models
 # pylint: disable=line-too-long
+for a list of available sentence_transformers models.
   columns: List of columns to be embedded.
   max_seq_length: Max sequence length to use for the model if applicable.
+  image_model: Whether the model is generating image embeddings.
   min_batch_size: The minimum batch size to be used for inference.
   max_batch_size: The maximum batch size to be used for inference.
   large_model: Whether to share the model across processes.
@@ -132,6 +138,7 @@ class SentenceTransformerEmbeddings(EmbeddingsManager):
 super().__init__(columns, **kwargs)
 self.model_name = model_name
 self.max_seq_length = max_seq_length
+self.image_model = image_model
 
   def get_model_handler(self):
 return _SentenceTransformerModelHandler(
@@ -144,8 +151,14 @@ class SentenceTransformerEmbeddings(EmbeddingsManager):
 large_model=self.large_model)
 
   def get_ptransform_for_processing(self, **kwargs) -> beam.PTransform:
-# wrap the model handler in a _TextEmbeddingHandler since
-# the SentenceTransformerEmbeddings works on text input data.
+# wrap the model handler in an appropriate embedding handler to provide
+# some type checking.
+if self.image_model:
+  return (
+  RunInference(
+  model_handler=_ImageEmbeddingHandler(self),
+  inference_args=self.inference_args,
+  ))
 return (
 RunInference(
 model_handler=_TextEmbeddingHandler(self),
diff --git 
a/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py 
b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py
index f94e747c5ed..d09a573b676 100644
--- a/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py
+++ b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py
@@ -35,6 +35,7 @@ from apache_beam.testing.util import equal_to
 try:
   from apache_beam.ml.transforms.embeddings.huggingface import 
SentenceTransformerEmbeddings
   from apache_beam.ml.transforms.embeddings.huggingface import 
InferenceAPIEmbeddings
+  from PIL import Image
   import torch
 except ImportError:
   SentenceTransformerEmbeddings = None  # type: ignore
@@ -46,10 +47,17 @@ try:
 except ImportError:
   tft = None
 
+# pylint: disable=ungrouped-imports
+try:
+  from PIL import Image
+except ImportError:
+  Image = None
+
 _HF_TOKEN = os.environ.get('HF_INFERENCE_TOKEN')
 test_query = "This is a test"
 test_query_column = "feature_1"
 DEFAULT_MODEL_NAME = "sentence-transformers/all-

(beam) branch master updated: Bump tqdm from 4.66.2 to 4.66.3 in /sdks/python/container/py38 (#31179)

2024-06-07 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 9edd413ef73 Bump tqdm from 4.66.2 to 4.66.3 in 
/sdks/python/container/py38 (#31179)
9edd413ef73 is described below

commit 9edd413ef734208e04e8be2b4e71be713b8a56a8
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
AuthorDate: Fri Jun 7 11:16:50 2024 -0400

Bump tqdm from 4.66.2 to 4.66.3 in /sdks/python/container/py38 (#31179)

Bumps [tqdm](https://github.com/tqdm/tqdm) from 4.66.2 to 4.66.3.
- [Release notes](https://github.com/tqdm/tqdm/releases)
- [Commits](https://github.com/tqdm/tqdm/compare/v4.66.2...v4.66.3)

---
updated-dependencies:
- dependency-name: tqdm
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] 
<49699333+dependabot[bot]@users.noreply.github.com>
---
 sdks/python/container/py38/base_image_requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdks/python/container/py38/base_image_requirements.txt 
b/sdks/python/container/py38/base_image_requirements.txt
index c87b4fac4b2..9cfd30ba6b3 100644
--- a/sdks/python/container/py38/base_image_requirements.txt
+++ b/sdks/python/container/py38/base_image_requirements.txt
@@ -151,7 +151,7 @@ tenacity==8.2.3
 testcontainers==3.7.1
 threadpoolctl==3.4.0
 tomli==2.0.1
-tqdm==4.66.2
+tqdm==4.66.3
 typing_extensions==4.11.0
 tzdata==2024.1
 tzlocal==5.2



(beam) branch master updated (10b6342c260 -> 47d9bca8bf1)

2024-06-07 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


from 10b6342c260 Fix assert message, fix streaming mode (#31540)
 add 47d9bca8bf1 Fix Python Linting Errors on Master (#31546)

No new revisions were added by this update.

Summary of changes:
 sdks/python/apache_beam/internal/gcp/auth.py  | 4 ++--
 sdks/python/apache_beam/internal/gcp/auth_test.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)



(beam) branch master updated: Fix assert message, fix streaming mode (#31540)

2024-06-07 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 10b6342c260 Fix assert message, fix streaming mode (#31540)
10b6342c260 is described below

commit 10b6342c26005d52963d410efc1a78e4bd03e888
Author: Vitaly Terentyev 
AuthorDate: Fri Jun 7 17:32:31 2024 +0400

Fix assert message, fix streaming mode (#31540)
---
 .../org/apache/beam/it/gcp/bigquery/BigQueryIOST.java | 19 ---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git 
a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java
 
b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java
index a49df8a..22ff94e293b 100644
--- 
a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java
+++ 
b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java
@@ -296,6 +296,11 @@ public final class BigQueryIOST extends IOStressTestBase {
 .withSchema(schema)
 
.withCustomGcsTempLocation(ValueProvider.StaticValueProvider.of(tempLocation)));
 
+String experiments =
+configuration.writeMethod.equals(STORAGE_API_AT_LEAST_ONCE_METHOD)
+? GcpOptions.STREAMING_ENGINE_EXPERIMENT + 
",streaming_mode_at_least_once"
+: GcpOptions.STREAMING_ENGINE_EXPERIMENT;
+
 PipelineLauncher.LaunchConfig options =
 PipelineLauncher.LaunchConfig.builder("write-bigquery")
 .setSdk(PipelineLauncher.Sdk.JAVA)
@@ -307,7 +312,7 @@ public final class BigQueryIOST extends IOStressTestBase {
 .toString())
 .addParameter("numWorkers", 
String.valueOf(configuration.numWorkers))
 .addParameter("maxNumWorkers", 
String.valueOf(configuration.maxNumWorkers))
-.addParameter("experiments", 
GcpOptions.STREAMING_ENGINE_EXPERIMENT)
+.addParameter("experiments", experiments)
 .build();
 
 PipelineLauncher.LaunchInfo launchInfo = pipelineLauncher.launch(project, 
region, options);
@@ -329,9 +334,17 @@ public final class BigQueryIOST extends IOStressTestBase {
 
 // Depending on writing method there might be duplicates on different 
sides (read or write).
 if (configuration.writeMethod.equals(STORAGE_API_AT_LEAST_ONCE_METHOD)) {
-  assertTrue(rowCount >= numRecords);
+  assertTrue(
+  String.format(
+  "Number of rows in the table (%d) is less than the expected 
number (%d). Missing records: %d",
+  rowCount, (long) numRecords, (long) numRecords - rowCount),
+  rowCount >= numRecords);
 } else {
-  assertTrue(numRecords >= rowCount);
+  assertTrue(
+  String.format(
+  "Number of rows in the table (%d) is greater than the expected 
number (%d).",
+  rowCount, (long) numRecords),
+  numRecords >= rowCount);
 }
 
 // export metrics



(beam) branch master updated: Deduplicate Base Embedding Handler Code (#31534)

2024-06-07 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 9a921d51054 Deduplicate Base Embedding Handler Code (#31534)
9a921d51054 is described below

commit 9a921d51054b7cf1be8119c76dabca80d26c9bfc
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Fri Jun 7 09:26:36 2024 -0400

Deduplicate Base Embedding Handler Code (#31534)

* Deduplicate Base Embedding Handler Code

* linting
---
 sdks/python/apache_beam/ml/transforms/base.py | 131 +-
 1 file changed, 43 insertions(+), 88 deletions(-)

diff --git a/sdks/python/apache_beam/ml/transforms/base.py 
b/sdks/python/apache_beam/ml/transforms/base.py
index 0d7db282bb8..678ab0882d2 100644
--- a/sdks/python/apache_beam/ml/transforms/base.py
+++ b/sdks/python/apache_beam/ml/transforms/base.py
@@ -587,9 +587,9 @@ class _MLTransformToPTransformMapper:
 return _transform_attribute_manager.load_attributes(artifact_location)
 
 
-class _TextEmbeddingHandler(ModelHandler):
+class _EmbeddingHandler(ModelHandler):
   """
-  A ModelHandler intended to be work on list[dict[str, str]] inputs.
+  A ModelHandler intended to be work on list[dict[str, Any]] inputs.
 
   The inputs to the model handler are expected to be a list of dicts.
 
@@ -597,12 +597,10 @@ class _TextEmbeddingHandler(ModelHandler):
   PCollection[E] to a PCollection[P], this ModelHandler would take a
   PCollection[Dict[str, E]] to a PCollection[Dict[str, P]].
 
-  _TextEmbeddingHandler will accept an EmbeddingsManager instance, which
+  _EmbeddingHandler will accept an EmbeddingsManager instance, which
   contains the details of the model to be loaded and the inference_fn to be
-  used. The purpose of _TextEmbeddingHandler is to generate embeddings for
-  text inputs using the EmbeddingsManager instance.
-
-  If the input is not a text column, a RuntimeError will be raised.
+  used. The purpose of _EmbeddingHandler is to generate embeddings for
+  general inputs using the EmbeddingsManager instance.
 
   This is an internal class and offers no backwards compatibility guarantees.
 
@@ -619,12 +617,9 @@ class _TextEmbeddingHandler(ModelHandler):
 return model
 
   def _validate_column_data(self, batch):
-if not isinstance(batch[0], (str, bytes)):
-  raise TypeError(
-  'Embeddings can only be generated on Dict[str, str].'
-  f'Got Dict[str, {type(batch[0])}] instead.')
+pass
 
-  def _validate_batch(self, batch: Sequence[Dict[str, List[str]]]):
+  def _validate_batch(self, batch: Sequence[Dict[str, Any]]):
 if not batch or not isinstance(batch[0], dict):
   raise TypeError(
   'Expected data to be dicts, got '
@@ -676,8 +671,7 @@ class _TextEmbeddingHandler(ModelHandler):
 
   def get_metrics_namespace(self) -> str:
 return (
-self._underlying.get_metrics_namespace() or
-'BeamML_TextEmbeddingHandler')
+self._underlying.get_metrics_namespace() or 'BeamML_EmbeddingHandler')
 
   def batch_elements_kwargs(self) -> Mapping[str, Any]:
 batch_sizes_map = {}
@@ -694,7 +688,41 @@ class _TextEmbeddingHandler(ModelHandler):
 pass
 
 
-class _ImageEmbeddingHandler(ModelHandler):
+class _TextEmbeddingHandler(_EmbeddingHandler):
+  """
+  A ModelHandler intended to be work on list[dict[str, str]] inputs.
+
+  The inputs to the model handler are expected to be a list of dicts.
+
+  For example, if the original mode is used with RunInference to take a
+  PCollection[E] to a PCollection[P], this ModelHandler would take a
+  PCollection[Dict[str, E]] to a PCollection[Dict[str, P]].
+
+  _TextEmbeddingHandler will accept an EmbeddingsManager instance, which
+  contains the details of the model to be loaded and the inference_fn to be
+  used. The purpose of _TextEmbeddingHandler is to generate embeddings for
+  text inputs using the EmbeddingsManager instance.
+
+  If the input is not a text column, a RuntimeError will be raised.
+
+  This is an internal class and offers no backwards compatibility guarantees.
+
+  Args:
+embeddings_manager: An EmbeddingsManager instance.
+  """
+  def _validate_column_data(self, batch):
+if not isinstance(batch[0], (str, bytes)):
+  raise TypeError(
+  'Embeddings can only be generated on Dict[str, str].'
+  f'Got Dict[str, {type(batch[0])}] instead.')
+
+  def get_metrics_namespace(self) -> str:
+return (
+self._underlying.get_metrics_namespace() or
+'BeamML_TextEmbeddingHandler')
+
+
+class _ImageEmbeddingHandler(_EmbeddingHandler):
   """
   A ModelHandler intended to be work on list[dict[str, Image]] inputs.
 
@@ -717,15 +745,6 @@ class _ImageEmbeddingHandler(ModelHandler):
   Args:
 embeddings_manager: An 

(beam) branch master updated (8c6e1a4654b -> 3cdbd3a908a)

2024-06-06 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


from 8c6e1a4654b Add missing import (#31533)
 add 3cdbd3a908a Implement Vertex AI Image Embedding MLTransform (#31514)

No new revisions were added by this update.

Summary of changes:
 .../ml/transforms/embeddings/vertex_ai.py  | 96 +-
 .../ml/transforms/embeddings/vertex_ai_test.py | 41 +
 2 files changed, 136 insertions(+), 1 deletion(-)



(beam) branch master updated: update the transform label error message (#31508)

2024-06-05 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new bd00a69402f update the transform label error message (#31508)
bd00a69402f is described below

commit bd00a69402f45520c193afd82bea51b31b90f681
Author: liferoad 
AuthorDate: Wed Jun 5 10:45:34 2024 -0400

update the transform label error message (#31508)
---
 sdks/python/apache_beam/pipeline.py  | 6 --
 sdks/python/apache_beam/pipeline_test.py | 6 --
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/sdks/python/apache_beam/pipeline.py 
b/sdks/python/apache_beam/pipeline.py
index 11bc74d27ec..a20fae2ae73 100644
--- a/sdks/python/apache_beam/pipeline.py
+++ b/sdks/python/apache_beam/pipeline.py
@@ -699,8 +699,10 @@ class Pipeline(HasDisplayData):
   else:
 raise RuntimeError(
 'A transform with label "%s" already exists in the pipeline. '
-'To apply a transform with a specified label write '
-'pvalue | "label" >> transform' % full_label)
+'To apply a transform with a specified label, write '
+'pvalue | "label" >> transform or use the option '
+'"auto_unique_labels" to automatically generate unique '
+'transform labels' % full_label)
 self.applied_labels.add(full_label)
 
 pvalueish, inputs = transform._extract_input_pvalues(pvalueish)
diff --git a/sdks/python/apache_beam/pipeline_test.py 
b/sdks/python/apache_beam/pipeline_test.py
index 38d8b1f03a1..a11f2e09ffc 100644
--- a/sdks/python/apache_beam/pipeline_test.py
+++ b/sdks/python/apache_beam/pipeline_test.py
@@ -264,8 +264,10 @@ class PipelineTest(unittest.TestCase):
 self.assertEqual(
 cm.exception.args[0],
 'A transform with label "CustomTransform" already exists in the '
-'pipeline. To apply a transform with a specified label write '
-'pvalue | "label" >> transform')
+'pipeline. To apply a transform with a specified label, write '
+'pvalue | "label" >> transform or use the option '
+'"auto_unique_labels" to automatically generate unique '
+'transform labels')
 
   def test_auto_unique_labels(self):
 



(beam) branch master updated: Add base _ImageEmbeddingHandler (#31501)

2024-06-05 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 7f51d6a9bdf Add base _ImageEmbeddingHandler (#31501)
7f51d6a9bdf is described below

commit 7f51d6a9bdf433158a8a6564a0154d8f16571847
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Wed Jun 5 10:43:46 2024 -0400

Add base _ImageEmbeddingHandler (#31501)

* Add base _ImageEmbeddingHandler

* add pillow dependency for testing

* Add skips if dependency not installed

* remove extra base dependencies

* fix import check

* further clean up image import

* linting

* typo
---
 sdks/python/apache_beam/ml/transforms/base.py  | 110 +
 sdks/python/apache_beam/ml/transforms/base_test.py |  88 +
 sdks/python/setup.py   |   1 +
 3 files changed, 199 insertions(+)

diff --git a/sdks/python/apache_beam/ml/transforms/base.py 
b/sdks/python/apache_beam/ml/transforms/base.py
index a297f62f5bc..0d7db282bb8 100644
--- a/sdks/python/apache_beam/ml/transforms/base.py
+++ b/sdks/python/apache_beam/ml/transforms/base.py
@@ -692,3 +692,113 @@ class _TextEmbeddingHandler(ModelHandler):
 
   def validate_inference_args(self, _):
 pass
+
+
+class _ImageEmbeddingHandler(ModelHandler):
+  """
+  A ModelHandler intended to be work on list[dict[str, Image]] inputs.
+
+  The inputs to the model handler are expected to be a list of dicts.
+
+  For example, if the original mode is used with RunInference to take a
+  PCollection[E] to a PCollection[P], this ModelHandler would take a
+  PCollection[Dict[str, E]] to a PCollection[Dict[str, P]].
+
+  _ImageEmbeddingHandler will accept an EmbeddingsManager instance, which
+  contains the details of the model to be loaded and the inference_fn to be
+  used. The purpose of _ImageEmbeddingHandler is to generate embeddings for
+  image inputs using the EmbeddingsManager instance.
+
+  If the input is not an Image representation column, a RuntimeError will be
+  raised.
+
+  This is an internal class and offers no backwards compatibility guarantees.
+
+  Args:
+embeddings_manager: An EmbeddingsManager instance.
+  """
+  def __init__(self, embeddings_manager: EmbeddingsManager):
+self.embedding_config = embeddings_manager
+self._underlying = self.embedding_config.get_model_handler()
+self.columns = self.embedding_config.get_columns_to_apply()
+
+  def load_model(self):
+model = self._underlying.load_model()
+return model
+
+  def _validate_column_data(self, batch):
+# Don't want to require framework-specific imports
+# here, so just catch columns of primatives for now.
+if isinstance(batch[0], (int, str, float, bool)):
+  raise TypeError(
+  'Embeddings can only be generated on Dict[str, Image].'
+  f'Got Dict[str, {type(batch[0])}] instead.')
+
+  def _validate_batch(self, batch: Sequence[Dict[str, List[Any]]]):
+if not batch or not isinstance(batch[0], dict):
+  raise TypeError(
+  'Expected data to be dicts, got '
+  f'{type(batch[0])} instead.')
+
+  def _process_batch(
+  self,
+  dict_batch: Dict[str, List[Any]],
+  model: ModelT,
+  inference_args: Optional[Dict[str, Any]]) -> Dict[str, List[Any]]:
+result: Dict[str, List[Any]] = collections.defaultdict(list)
+input_keys = dict_batch.keys()
+missing_columns_in_data = set(self.columns) - set(input_keys)
+if missing_columns_in_data:
+  raise RuntimeError(
+  f'Data does not contain the following columns '
+  f': {missing_columns_in_data}.')
+for key, batch in dict_batch.items():
+  if key in self.columns:
+self._validate_column_data(batch)
+prediction = self._underlying.run_inference(
+batch, model, inference_args)
+if isinstance(prediction, np.ndarray):
+  prediction = prediction.tolist()
+  result[key] = prediction  # type: ignore[assignment]
+else:
+  result[key] = prediction  # type: ignore[assignment]
+  else:
+result[key] = batch
+return result
+
+  def run_inference(
+  self,
+  batch: Sequence[Dict[str, List[str]]],
+  model: ModelT,
+  inference_args: Optional[Dict[str, Any]] = None,
+  ) -> List[Dict[str, Union[List[float], List[str:
+"""
+Runs inference on a batch of text inputs. The inputs are expected to be
+a list of dicts. Each dict should have the same keys, and the shape
+should be of the same size for a single key across the batch.
+"""
+self._validate_batch(batch)
+dict_batch = _convert_list_of_dicts_to_dict_of_lists(list_of_dicts=batch)
+transformed_ba

(beam) branch master updated: Add ApplyBucketsWithInterpolation TFTransform (#31291)

2024-05-29 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 06e103d87e8 Add ApplyBucketsWithInterpolation TFTransform (#31291)
06e103d87e8 is described below

commit 06e103d87e8ac883f606475dbadbefed4ba77c9a
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Wed May 29 14:23:57 2024 -0400

Add ApplyBucketsWithInterpolation TFTransform (#31291)

* Add ApplyBucketsWithInterpolation TFTransform

* Update sdks/python/apache_beam/ml/transforms/tft.py

Co-authored-by: tvalentyn 

* add tft documentation link

* change docstring wording around bucket_boundaries

* Update sdks/python/apache_beam/ml/transforms/tft.py

Co-authored-by: tvalentyn 

-

Co-authored-by: tvalentyn 
---
 sdks/python/apache_beam/ml/transforms/tft.py  | 49 ---
 sdks/python/apache_beam/ml/transforms/tft_test.py | 30 ++
 2 files changed, 74 insertions(+), 5 deletions(-)

diff --git a/sdks/python/apache_beam/ml/transforms/tft.py 
b/sdks/python/apache_beam/ml/transforms/tft.py
index 370043bc0d9..e2f02971e7c 100644
--- a/sdks/python/apache_beam/ml/transforms/tft.py
+++ b/sdks/python/apache_beam/ml/transforms/tft.py
@@ -337,16 +337,16 @@ class ApplyBuckets(TFTOperation):
   name: Optional[str] = None):
 """
 This functions is used to map the element to a positive index i for
-which bucket_boundaries[i-1] <= element < bucket_boundaries[i],
-if it exists. If input < bucket_boundaries[0], then element is
-mapped to 0. If element >= bucket_boundaries[-1], then element is
+which `bucket_boundaries[i-1] <= element < bucket_boundaries[i]`,
+if it exists. If `input < bucket_boundaries[0]`, then element is
+mapped to 0. If `element >= bucket_boundaries[-1]`, then element is
 mapped to len(bucket_boundaries). NaNs are mapped to
 len(bucket_boundaries).
 
 Args:
   columns: A list of column names to apply the transformation on.
-  bucket_boundaries: A rank 2 Tensor or list representing the bucket
-boundaries sorted in ascending order.
+  bucket_boundaries: An iterable of ints or floats representing the bucket
+boundaries. Must be sorted in ascending order.
   name: (Optional) A string that specifies the name of the operation.
 """
 super().__init__(columns)
@@ -363,6 +363,45 @@ class ApplyBuckets(TFTOperation):
 return output
 
 
+@register_input_dtype(float)
+class ApplyBucketsWithInterpolation(TFTOperation):
+  def __init__(
+  self,
+  columns: List[str],
+  bucket_boundaries: Iterable[Union[int, float]],
+  name: Optional[str] = None):
+"""Interpolates values within the provided buckets and then normalizes to
+[0, 1].
+
+Input values are bucketized based on the provided boundaries such that the
+input is mapped to a positive index i for which `bucket_boundaries[i-1] <=
+element < bucket_boundaries[i]`, if it exists. The values are then
+normalized to the range [0,1] within the bucket, with NaN values being
+mapped to 0.5.
+
+For more information, see:
+
https://www.tensorflow.org/tfx/transform/api_docs/python/tft/apply_buckets_with_interpolation
+
+Args:
+  columns: A list of column names to apply the transformation on.
+  bucket_boundaries: An iterable of ints or floats representing the bucket
+boundaries sorted in ascending order.
+  name: (Optional) A string that specifies the name of the operation.
+"""
+super().__init__(columns)
+self.bucket_boundaries = [bucket_boundaries]
+self.name = name
+
+  def apply_transform(
+  self, data: common_types.TensorType,
+  output_column_name: str) -> Dict[str, common_types.TensorType]:
+output = {
+output_column_name: tft.apply_buckets_with_interpolation(
+x=data, bucket_boundaries=self.bucket_boundaries, name=self.name)
+}
+return output
+
+
 @register_input_dtype(float)
 class Bucketize(TFTOperation):
   def __init__(
diff --git a/sdks/python/apache_beam/ml/transforms/tft_test.py 
b/sdks/python/apache_beam/ml/transforms/tft_test.py
index 5c42ecc012f..f5615e9d4ad 100644
--- a/sdks/python/apache_beam/ml/transforms/tft_test.py
+++ b/sdks/python/apache_beam/ml/transforms/tft_test.py
@@ -364,6 +364,36 @@ class ApplyBucketsTest(unittest.TestCase):
   actual_output, equal_to(expected_output, equals_fn=np.array_equal))
 
 
+class ApplyBucketsWithInterpolationTest(unittest.TestCase):
+  def setUp(self) -> None:
+self.artifact_location = tempfile.mkdtemp()
+
+  def tearDown(self):
+shutil.rmtree(self.artifact_location)
+
+  @parameterize

(beam) branch master updated: [Go SDK] Make it clearer that timers and data don't interact negatively in element batches. (#31319)

2024-05-21 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 0e734fe1c38 [Go SDK] Make it clearer that timers and data don't 
interact negatively in element batches. (#31319)
0e734fe1c38 is described below

commit 0e734fe1c383b09096988b80551843374776c18f
Author: Robert Burke 
AuthorDate: Tue May 21 07:11:07 2024 -0700

[Go SDK] Make it clearer that timers and data don't interact negatively in 
element batches. (#31319)

* Don't swallow data errors on timer errors.

* reset error in case of timer handling

* Don't drop timers on splits.

-

Co-authored-by: lostluck <13907733+lostl...@users.noreply.github.com>
---
 sdks/go/pkg/beam/core/runtime/exec/datasource.go | 33 +++-
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasource.go 
b/sdks/go/pkg/beam/core/runtime/exec/datasource.go
index 674de44cf35..156d9565379 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/datasource.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/datasource.go
@@ -107,15 +107,10 @@ func (n *DataSource) StartBundle(ctx context.Context, id 
string, data DataContex
return n.Out.StartBundle(ctx, id, data)
 }
 
-// errSplitSuccess is a marker error to indicate we've reached the split index.
-// Akin to io.EOF.
-var errSplitSuccess = errors.New("split index reached")
-
 // process handles converting elements from the data source to timers.
 //
 // The data and timer callback functions must return an io.EOF if the reader 
terminates to signal that an additional
-// buffer is desired. On successful splits, [splitSuccess] must be returned to 
indicate that the
-// PTransform is done processing data for this instruction.
+// buffer is desired.
 func (n *DataSource) process(ctx context.Context, data func(bcr 
*byteCountReader, ptransformID string) error, timer func(bcr *byteCountReader, 
ptransformID, timerFamilyID string) error) error {
// The SID contains this instruction's expected data processing 
transform (this one).
elms, err := n.source.OpenElementChan(ctx, n.SID, 
maps.Keys(n.OnTimerTransforms))
@@ -129,7 +124,6 @@ func (n *DataSource) process(ctx context.Context, data 
func(bcr *byteCountReader
var byteCount int
bcr := byteCountReader{reader: , count: }
 
-   splitPrimaryComplete := map[string]bool{}
for {
var err error
select {
@@ -138,24 +132,21 @@ func (n *DataSource) process(ctx context.Context, data 
func(bcr *byteCountReader
if !ok {
return nil
}
-   if splitPrimaryComplete[e.PtransformID] {
-   continue
-   }
if len(e.Data) > 0 {
r.Reset(e.Data)
err = data(, e.PtransformID)
}
+   if err != nil && err != io.EOF {
+   return errors.Wrapf(err, "source failed 
processing data")
+   }
+   // Process any simultaneously sent timers.
+   // If the data channel has split though
if len(e.Timers) > 0 {
r.Reset(e.Timers)
err = timer(, e.PtransformID, 
e.TimerFamilyID)
}
-
-   if err == errSplitSuccess {
-   // Returning splitSuccess means we've split, 
and aren't consuming the remaining buffer.
-   // We mark the PTransform done to ignore 
further data.
-   splitPrimaryComplete[e.PtransformID] = true
-   } else if err != nil && err != io.EOF {
-   return errors.Wrap(err, "source failed")
+   if err != nil && err != io.EOF {
+   return errors.Wrap(err, "source failed 
processing timers")
}
// io.EOF means the reader successfully drained.
// We're ready for a new buffer.
@@ -210,8 +201,13 @@ func (n *DataSource) Process(ctx context.Context) 
([]*Checkpoint, error) {
cp = MakeElementDecoder(c)
}
 
+   hasSplit := map[string]bool{}
var checkpoints []*Checkpoint
err := n.process(ctx, func(bcr *byteCountReader, ptransformID string) 
error {
+   // Check if this transform has already successfully, and if so, 
skip reading and decoding of the elements in the buffer.
+ 

(beam) branch master updated: Implement DeduplicateTensorPerRow in MLTransform (#31307)

2024-05-16 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 118c51404d9 Implement DeduplicateTensorPerRow in MLTransform (#31307)
118c51404d9 is described below

commit 118c51404d93132d4a0cae135f00ff68d4fd84bf
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Thu May 16 10:29:04 2024 -0400

Implement DeduplicateTensorPerRow in MLTransform (#31307)
---
 sdks/python/apache_beam/ml/transforms/tft.py  | 22 
 sdks/python/apache_beam/ml/transforms/tft_test.py | 62 +++
 2 files changed, 84 insertions(+)

diff --git a/sdks/python/apache_beam/ml/transforms/tft.py 
b/sdks/python/apache_beam/ml/transforms/tft.py
index 550dbedbc7b..370043bc0d9 100644
--- a/sdks/python/apache_beam/ml/transforms/tft.py
+++ b/sdks/python/apache_beam/ml/transforms/tft.py
@@ -681,3 +681,25 @@ class HashStrings(TFTOperation):
 name=self.name)
 }
 return output_dict
+
+
+@register_input_dtype(str)
+class DeduplicateTensorPerRow(TFTOperation):
+  def __init__(self, columns: List[str], name: Optional[str] = None):
+""" Deduplicates each row (0th dimension) of the provided tensor.
+
+Args:
+  columns: A list of the columns to apply the transformation on.
+  name: optional. A name for this operation. 
+"""
+self.name = name
+super().__init__(columns)
+
+  def apply_transform(
+  self, data: common_types.TensorType,
+  output_col_name: str) -> Dict[str, common_types.TensorType]:
+output_dict = {
+output_col_name: tft.deduplicate_tensor_per_row(
+input_tensor=data, name=self.name)
+}
+return output_dict
diff --git a/sdks/python/apache_beam/ml/transforms/tft_test.py 
b/sdks/python/apache_beam/ml/transforms/tft_test.py
index 6763032a8eb..5c42ecc012f 100644
--- a/sdks/python/apache_beam/ml/transforms/tft_test.py
+++ b/sdks/python/apache_beam/ml/transforms/tft_test.py
@@ -1009,5 +1009,67 @@ class HashWordsTest(unittest.TestCase):
   assert_that(result, equal_to(expected_values, equals_fn=np.array_equal))
 
 
+class DeduplicateTensorPerRowTest(unittest.TestCase):
+  def setUp(self) -> None:
+self.artifact_location = tempfile.mkdtemp()
+
+  def tearDown(self):
+shutil.rmtree(self.artifact_location)
+
+  def test_deduplicate(self):
+values = [{
+'x': [b'a', b'b', b'a', b'b'],
+}, {
+'x': [b'b', b'c', b'b', b'c']
+}]
+
+expected_output = [np.array([b'a', b'b']), np.array([b'b', b'c'])]
+with beam.Pipeline() as p:
+  list_result = (
+  p
+  | "listCreate" >> beam.Create(values)
+  | "listMLTransform" >> base.MLTransform(
+  write_artifact_location=self.artifact_location).with_transform(
+  tft.DeduplicateTensorPerRow(columns=['x'])))
+  result = (list_result | beam.Map(lambda x: x.x))
+  assert_that(result, equal_to(expected_output, equals_fn=np.array_equal))
+
+  def test_deduplicate_no_op(self):
+values = [{
+'x': [b'a', b'b'],
+}, {
+'x': [b'c', b'd']
+}]
+
+expected_output = [np.array([b'a', b'b']), np.array([b'c', b'd'])]
+with beam.Pipeline() as p:
+  list_result = (
+  p
+  | "listCreate" >> beam.Create(values)
+  | "listMLTransform" >> base.MLTransform(
+  write_artifact_location=self.artifact_location).with_transform(
+  tft.DeduplicateTensorPerRow(columns=['x'])))
+  result = (list_result | beam.Map(lambda x: x.x))
+  assert_that(result, equal_to(expected_output, equals_fn=np.array_equal))
+
+  def test_deduplicate_different_output_sizes(self):
+values = [{
+'x': [b'a', b'b', b'a', b'b'],
+}, {
+'x': [b'c', b'a', b'd', b'd']
+}]
+
+expected_output = [np.array([b'a', b'b']), np.array([b'c', b'a', b'd'])]
+with beam.Pipeline() as p:
+  list_result = (
+  p
+  | "listCreate" >> beam.Create(values)
+  | "listMLTransform" >> base.MLTransform(
+  write_artifact_location=self.artifact_location).with_transform(
+  tft.DeduplicateTensorPerRow(columns=['x'])))
+  result = (list_result | beam.Map(lambda x: x.x))
+  assert_that(result, equal_to(expected_output, equals_fn=np.array_equal))
+
+
 if __name__ == '__main__':
   unittest.main()



(beam) branch master updated: Implement the hash_words TFT operation (#31249)

2024-05-13 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new c230655a1e5 Implement the hash_words TFT operation (#31249)
c230655a1e5 is described below

commit c230655a1e599e25a69b77d8a85fad45e5fc7587
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Mon May 13 14:22:25 2024 -0400

Implement the hash_words TFT operation (#31249)

* Implement the hash-words TFT operation

* linting

* Update sdks/python/apache_beam/ml/transforms/tft.py

Co-authored-by: tvalentyn 

* tighten key type hint

* formatting

-

Co-authored-by: tvalentyn 
---
 sdks/python/apache_beam/ml/transforms/tft.py  | 44 +
 sdks/python/apache_beam/ml/transforms/tft_test.py | 77 +++
 2 files changed, 121 insertions(+)

diff --git a/sdks/python/apache_beam/ml/transforms/tft.py 
b/sdks/python/apache_beam/ml/transforms/tft.py
index 9b02cf8b75c..550dbedbc7b 100644
--- a/sdks/python/apache_beam/ml/transforms/tft.py
+++ b/sdks/python/apache_beam/ml/transforms/tft.py
@@ -637,3 +637,47 @@ class BagOfWords(TFTOperation):
 def count_unique_words(
 data: tf.SparseTensor, output_vocab_name: Optional[str]) -> None:
   tft.count_per_key(data, key_vocabulary_filename=output_vocab_name)
+
+
+@register_input_dtype(str)
+class HashStrings(TFTOperation):
+  def __init__(
+  self,
+  columns: List[str],
+  hash_buckets: int,
+  key: Optional[Tuple[int, int]] = None,
+  name: Optional[str] = None):
+'''Hashes strings into the provided number of buckets.
+
+Args:
+  columns: A list of the column names to apply the transformation on.
+  hash_buckets: the number of buckets to hash the strings into.
+  key: optional. An array of two Python `uint64`. If passed, output will be
+a deterministic function of `strings` and `key`. Note that hashing will
+be slower if this value is specified.
+  name: optional. A name for this operation.
+
+Raises:
+  ValueError if `hash_buckets` is not a positive and non-zero integer.
+'''
+self.hash_buckets = hash_buckets
+self.key = key
+self.name = name
+
+if hash_buckets < 1:
+  raise ValueError(
+  'number of hash buckets must be positive, got ', hash_buckets)
+
+super().__init__(columns)
+
+  def apply_transform(
+  self, data: common_types.TensorType,
+  output_col_name: str) -> Dict[str, common_types.TensorType]:
+output_dict = {
+output_col_name: tft.hash_strings(
+strings=data,
+hash_buckets=self.hash_buckets,
+key=self.key,
+name=self.name)
+}
+return output_dict
diff --git a/sdks/python/apache_beam/ml/transforms/tft_test.py 
b/sdks/python/apache_beam/ml/transforms/tft_test.py
index ed7c301f3b8..6763032a8eb 100644
--- a/sdks/python/apache_beam/ml/transforms/tft_test.py
+++ b/sdks/python/apache_beam/ml/transforms/tft_test.py
@@ -932,5 +932,82 @@ class BagOfWordsTest(unittest.TestCase):
 self.assertEqual(expected_data, actual_data)
 
 
+class HashWordsTest(unittest.TestCase):
+  def setUp(self) -> None:
+self.artifact_location = tempfile.mkdtemp()
+
+  def tearDown(self):
+shutil.rmtree(self.artifact_location)
+
+  def test_single_bucket(self):
+list_data = [{'x': 'this is a test string'}]
+expected_values = [np.array([0])]
+with beam.Pipeline() as p:
+  list_result = (
+  p
+  | "listCreate" >> beam.Create(list_data)
+  | "listMLTransform" >> base.MLTransform(
+  write_artifact_location=self.artifact_location).with_transform(
+  tft.HashStrings(columns=['x'], hash_buckets=1)))
+  result = (list_result | beam.Map(lambda x: x.x))
+  assert_that(result, equal_to(expected_values, equals_fn=np.array_equal))
+
+  def test_multi_bucket_one_string(self):
+list_data = [{'x': 'this is a test string'}]
+expected_values = [np.array([1])]
+with beam.Pipeline() as p:
+  list_result = (
+  p
+  | "listCreate" >> beam.Create(list_data)
+  | "listMLTransform" >> base.MLTransform(
+  write_artifact_location=self.artifact_location).with_transform(
+  tft.HashStrings(columns=['x'], hash_buckets=2)))
+  result = (list_result | beam.Map(lambda x: x.x))
+  assert_that(result, equal_to(expected_values, equals_fn=np.array_equal))
+
+  def test_one_bucket_multi_string(self):
+list_data = [{'x': ['these', 'are', 'test', 'strings']}]
+expected_values = [np.array([0, 0, 0, 0])]
+with beam.Pipeline() as p:
+  list_result = (
+  p
+  | "l

(beam) branch master updated: Avoid oversizing batch sizes with size estimation function (#31228)

2024-05-10 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 2d53926542f Avoid oversizing batch sizes with size estimation function 
(#31228)
2d53926542f is described below

commit 2d53926542f82a8b955eb541f13475f9bef091a7
Author: Danny McCormick 
AuthorDate: Fri May 10 10:12:17 2024 -0400

Avoid oversizing batch sizes with size estimation function (#31228)

* Avoid oversizing batch sizes with size estimation function

* lint
---
 sdks/python/apache_beam/transforms/util.py  | 18 --
 sdks/python/apache_beam/transforms/util_test.py | 33 ++---
 2 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/sdks/python/apache_beam/transforms/util.py 
b/sdks/python/apache_beam/transforms/util.py
index edf79b7c798..750d98f0789 100644
--- a/sdks/python/apache_beam/transforms/util.py
+++ b/sdks/python/apache_beam/transforms/util.py
@@ -579,14 +579,15 @@ class _GlobalWindowsBatchingDoFn(DoFn):
 self._batch_size_estimator.ignore_next_timing()
 
   def process(self, element):
-self._batch.append(element)
-self._running_batch_size += self._element_size_fn(element)
-if self._running_batch_size >= self._target_batch_size:
+element_size = self._element_size_fn(element)
+if self._running_batch_size + element_size > self._target_batch_size:
   with self._batch_size_estimator.record_time(self._running_batch_size):
 yield window.GlobalWindows.windowed_value_at_end_of_window(self._batch)
   self._batch = []
   self._running_batch_size = 0
   self._target_batch_size = self._batch_size_estimator.next_batch_size()
+self._batch.append(element)
+self._running_batch_size += element_size
 
   def finish_bundle(self):
 if self._batch:
@@ -621,15 +622,18 @@ class _WindowAwareBatchingDoFn(DoFn):
 
   def process(self, element, window=DoFn.WindowParam):
 batch = self._batches[window]
-batch.elements.append(element)
-batch.size += self._element_size_fn(element)
-if batch.size >= self._target_batch_size:
+element_size = self._element_size_fn(element)
+if batch.size + element_size > self._target_batch_size:
   with self._batch_size_estimator.record_time(batch.size):
 yield windowed_value.WindowedValue(
 batch.elements, window.max_timestamp(), (window, ))
   del self._batches[window]
   self._target_batch_size = self._batch_size_estimator.next_batch_size()
-elif len(self._batches) > self._MAX_LIVE_WINDOWS:
+
+self._batches[window].elements.append(element)
+self._batches[window].size += element_size
+
+if len(self._batches) > self._MAX_LIVE_WINDOWS:
   window, batch = max(
   self._batches.items(),
   key=lambda window_batch: window_batch[1].size)
diff --git a/sdks/python/apache_beam/transforms/util_test.py 
b/sdks/python/apache_beam/transforms/util_test.py
index 53898d57998..74d9f438a5d 100644
--- a/sdks/python/apache_beam/transforms/util_test.py
+++ b/sdks/python/apache_beam/transforms/util_test.py
@@ -299,15 +299,40 @@ class BatchElementsTest(unittest.TestCase):
   res = (
   p
   | beam.Create([
-  'a', 'a', 'aa',  # First batch.
-  'aa', 'a',   # Second batch.
-  'a', 'aaa', 'a', 'a' # Third batch.
+  'a', 'a',# First batch.
+  'aa',# Second batch.
+  'a', 'a',# Third batch.
+  'a', 'aaa', 'a', 'a' # Fourth batch.
   ], reshuffle=False)
   | util.BatchElements(
   min_batch_size=10, max_batch_size=10, element_size_fn=len)
   | beam.Map(lambda batch: ''.join(batch))
   | beam.Map(len))
-  assert_that(res, equal_to([12, 11, 10]))
+  assert_that(res, equal_to([2, 10, 10, 10]))
+
+  def test_sized_windowed_batches(self):
+# Assumes a single bundle, in order...
+with TestPipeline() as p:
+  res = (
+  p
+  | beam.Create(range(1, 8), reshuffle=False)
+  | beam.Map(lambda t: window.TimestampedValue('a' * t, t))
+  | beam.WindowInto(window.FixedWindows(3))
+  | util.BatchElements(
+  min_batch_size=11,
+  max_batch_size=11,
+  element_size_fn=len,
+  clock=FakeClock())
+  | beam.Map(lambda batch: ''.join(batch)))
+  assert_that(
+  res,
+  equal_to([
+  'a' * (1+2), # Elements in [1, 3)
+  'a' * (3+4), # Elements in [3, 6)
+  'a' * 5,
+  'a' * 6, # Elements in [6, 9)
+  'a' * 7,
+  ]))
 
   def test_target_duration(self):
 clock = FakeClock()



(beam) branch master updated (20409b5204d -> faaa68c1a1c)

2024-05-08 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


from 20409b5204d Fix python_test broke by #31199 (#31221)
 add faaa68c1a1c Fix lint warning: JobId vs JobID (#31197)

No new revisions were added by this update.

Summary of changes:
 sdks/go/pkg/beam/runners/prism/internal/web/web.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)



(beam) branch master updated: Bump jinja2 from 2.11.3 to 3.1.4 in /.test-infra/jenkins/metrics_report (#31191)

2024-05-06 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new a22ee682fa8 Bump jinja2 from 2.11.3 to 3.1.4 in 
/.test-infra/jenkins/metrics_report (#31191)
a22ee682fa8 is described below

commit a22ee682fa87a126a1efd2f3ddb3c33913506654
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
AuthorDate: Mon May 6 15:08:39 2024 -0400

Bump jinja2 from 2.11.3 to 3.1.4 in /.test-infra/jenkins/metrics_report 
(#31191)

Bumps [jinja2](https://github.com/pallets/jinja) from 2.11.3 to 3.1.4.
- [Release notes](https://github.com/pallets/jinja/releases)
- [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/jinja/compare/2.11.3...3.1.4)

---
updated-dependencies:
- dependency-name: jinja2
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] 
<49699333+dependabot[bot]@users.noreply.github.com>
---
 .test-infra/jenkins/metrics_report/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.test-infra/jenkins/metrics_report/requirements.txt 
b/.test-infra/jenkins/metrics_report/requirements.txt
index 4716e65ddb9..e40f8d92982 100644
--- a/.test-infra/jenkins/metrics_report/requirements.txt
+++ b/.test-infra/jenkins/metrics_report/requirements.txt
@@ -20,5 +20,5 @@
 # https://github.com/aws/aws-sam-cli/issues/3661
 markupsafe==2.0.1
 influxdb==5.3.0
-Jinja2==2.11.3
+Jinja2==3.1.4
 prettytable==0.7.2



(beam) branch master updated: Pass location for Vertex endpoint retrieval (#31171)

2024-05-03 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 7dbdbdfbad4 Pass location for Vertex endpoint retrieval (#31171)
7dbdbdfbad4 is described below

commit 7dbdbdfbad45f50eab1b1de7f5589b3f28903934
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Fri May 3 12:50:07 2024 -0400

Pass location for Vertex endpoint retrieval (#31171)
---
 .../apache_beam/ml/inference/vertex_ai_inference.py   | 15 ++-
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py 
b/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py
index 95660441a84..4c4163accfb 100644
--- a/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py
+++ b/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py
@@ -133,9 +133,11 @@ class VertexAIModelHandlerJSON(ModelHandler[Any,
 # Check for liveness here but don't try to actually store the endpoint
 # in the class yet
 self.endpoint_name = endpoint_id
+self.location = location
 self.is_private = private
 
-_ = self._retrieve_endpoint(self.endpoint_name, self.is_private)
+_ = self._retrieve_endpoint(
+self.endpoint_name, self.location, self.is_private)
 
 # Configure AdaptiveThrottler and throttling metrics for client-side
 # throttling behavior.
@@ -147,7 +149,8 @@ class VertexAIModelHandlerJSON(ModelHandler[Any,
 window_ms=1, bucket_ms=1, overload_ratio=2)
 
   def _retrieve_endpoint(
-  self, endpoint_id: str, is_private: bool) -> aiplatform.Endpoint:
+  self, endpoint_id: str, location: str,
+  is_private: bool) -> aiplatform.Endpoint:
 """Retrieves an AI Platform endpoint and queries it for liveness/deployed
 models.
 
@@ -162,10 +165,11 @@ class VertexAIModelHandlerJSON(ModelHandler[Any,
 """
 if is_private:
   endpoint: aiplatform.Endpoint = aiplatform.PrivateEndpoint(
-  endpoint_name=endpoint_id)
+  endpoint_name=endpoint_id, location=location)
   LOGGER.debug("Treating endpoint %s as private", endpoint_id)
 else:
-  endpoint = aiplatform.Endpoint(endpoint_name=endpoint_id)
+  endpoint = aiplatform.Endpoint(
+  endpoint_name=endpoint_id, location=location)
   LOGGER.debug("Treating endpoint %s as public", endpoint_id)
 
 try:
@@ -185,7 +189,8 @@ class VertexAIModelHandlerJSON(ModelHandler[Any,
 """
 # Check to make sure the endpoint is still active since pipeline
 # construction time
-ep = self._retrieve_endpoint(self.endpoint_name, self.is_private)
+ep = self._retrieve_endpoint(
+self.endpoint_name, self.location, self.is_private)
 return ep
 
   @retry.with_exponential_backoff(



(beam) branch master updated: Pin tensorflow version in notebook (#31145)

2024-05-01 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 44e2abea788 Pin tensorflow version in notebook (#31145)
44e2abea788 is described below

commit 44e2abea7887cd692ef7e5623c632442858c51ba
Author: Danny McCormick 
AuthorDate: Wed May 1 13:57:03 2024 -0400

Pin tensorflow version in notebook (#31145)
---
 examples/notebooks/beam-ml/automatic_model_refresh.ipynb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/notebooks/beam-ml/automatic_model_refresh.ipynb 
b/examples/notebooks/beam-ml/automatic_model_refresh.ipynb
index 54fb16b6ef0..5b5d2ed484c 100644
--- a/examples/notebooks/beam-ml/automatic_model_refresh.ipynb
+++ b/examples/notebooks/beam-ml/automatic_model_refresh.ipynb
@@ -131,7 +131,7 @@
}],
   "source": [
 "!pip install apache_beam[gcp]>=2.46.0 --quiet\n",
-"!pip install tensorflow --quiet\n",
+"!pip install tensorflow==2.15.0 --quiet\n",
 "!pip install tensorflow_hub --quiet"
   ]
 },
@@ -272,7 +272,7 @@
   "cell_type": "code",
   "source": [
 "# In a requirements file, define the dependencies required for the 
pipeline.\n",
-"!printf 'tensorflow>=2.12.0\\ntensorflow_hub>=0.10.0\\nPillow>=9.0.0' 
> ./requirements.txt\n",
+"!printf 'tensorflow==2.15.0\\ntensorflow_hub>=0.10.0\\nPillow>=9.0.0' 
> ./requirements.txt\n",
 "# Install the pipeline dependencies on Dataflow.\n",
 "options.view_as(SetupOptions).requirements_file = 
'./requirements.txt'"
   ],



(beam) branch master updated (de665b77d9f -> ba7ea856ffe)

2024-04-23 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


from de665b77d9f [YAML] - Normalize YAML PubSub format (#31068)
 add ba7ea856ffe Revert "Change type for UnboundedReaderMaxReadTimeSec 
(#31037)" (#31082)

No new revisions were added by this update.

Summary of changes:
 .../runners/dataflow/options/DataflowPipelineDebugOptions.java   | 9 -
 .../apache/beam/runners/dataflow/worker/WorkerCustomSources.java | 4 +---
 .../beam/runners/dataflow/worker/WorkerCustomSourcesTest.java| 7 +++
 3 files changed, 8 insertions(+), 12 deletions(-)



(beam) 01/01: Revert "Change type for UnboundedReaderMaxReadTimeSec (#31037)"

2024-04-23 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch 
revert-31037-readerMaxReadTimeSec_as_double
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 5c8f548c944a9bbc32e07a13eefaac5493b26808
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Tue Apr 23 10:42:04 2024 -0400

Revert "Change type for UnboundedReaderMaxReadTimeSec (#31037)"

This reverts commit bb310e7e90720b620f1089574f1656ca84a3656d.
---
 .../runners/dataflow/options/DataflowPipelineDebugOptions.java   | 9 -
 .../apache/beam/runners/dataflow/worker/WorkerCustomSources.java | 4 +---
 .../beam/runners/dataflow/worker/WorkerCustomSourcesTest.java| 7 +++
 3 files changed, 8 insertions(+), 12 deletions(-)

diff --git 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
index 3f6c47ece68..30496dec296 100644
--- 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
+++ 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
@@ -218,12 +218,11 @@ public interface DataflowPipelineDebugOptions
 
   /** The max amount of time an UnboundedReader is consumed before 
checkpointing. */
   @Description(
-  "The max amount of time before an UnboundedReader is consumed before 
checkpointing, "
-  + "in seconds. Duration can be set to fractions of seconds. ")
-  @Default.Double(10.0)
-  double getUnboundedReaderMaxReadTimeSec();
+  "The max amount of time before an UnboundedReader is consumed before 
checkpointing, in seconds.")
+  @Default.Integer(10)
+  Integer getUnboundedReaderMaxReadTimeSec();
 
-  void setUnboundedReaderMaxReadTimeSec(double value);
+  void setUnboundedReaderMaxReadTimeSec(Integer value);
 
   /** The max elements read from an UnboundedReader before checkpointing. */
   @Description("The max elements read from an UnboundedReader before 
checkpointing. ")
diff --git 
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
 
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
index a8e358f19e0..8c086016ee9 100644
--- 
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
+++ 
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
@@ -798,9 +798,7 @@ public class WorkerCustomSources {
   DataflowPipelineDebugOptions debugOptions = 
options.as(DataflowPipelineDebugOptions.class);
   this.endTime =
   Instant.now()
-  .plus(
-  Duration.millis(
-  (long) (debugOptions.getUnboundedReaderMaxReadTimeSec() 
* 1000L)));
+  
.plus(Duration.standardSeconds(debugOptions.getUnboundedReaderMaxReadTimeSec()));
   this.maxElems = debugOptions.getUnboundedReaderMaxElements();
   this.backoffFactory =
   FluentBackoff.DEFAULT
diff --git 
a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
 
b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
index 261567930fe..d451ec093f7 100644
--- 
a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
+++ 
b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
@@ -598,7 +598,6 @@ public class WorkerCustomSourcesTest {
 int maxElements = 10;
 DataflowPipelineDebugOptions debugOptions = 
options.as(DataflowPipelineDebugOptions.class);
 debugOptions.setUnboundedReaderMaxElements(maxElements);
-debugOptions.setUnboundedReaderMaxReadTimeSec(10);
 
 ByteString state = ByteString.EMPTY;
 for (int i = 0; i < 10 * maxElements;
@@ -646,10 +645,10 @@ public class WorkerCustomSourcesTest {
 numReadOnThisIteration++;
   }
   Instant afterReading = Instant.now();
-  double maxReadSec = debugOptions.getUnboundedReaderMaxReadTimeSec();
+  long maxReadSec = debugOptions.getUnboundedReaderMaxReadTimeSec();
   assertThat(
-  new Duration(beforeReading, afterReading).getMillis(),
-  lessThanOrEqualTo((long) ((maxReadSec + 1) * 1000L)));
+  new Duration(beforeReading, afterReading).getStandardSeconds(),
+  lessThanOrEqualTo(maxReadSec + 1));
   assert

(beam) branch revert-31037-readerMaxReadTimeSec_as_double created (now 5c8f548c944)

2024-04-23 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch 
revert-31037-readerMaxReadTimeSec_as_double
in repository https://gitbox.apache.org/repos/asf/beam.git


  at 5c8f548c944 Revert "Change type for UnboundedReaderMaxReadTimeSec 
(#31037)"

This branch includes the following new commits:

 new 5c8f548c944 Revert "Change type for UnboundedReaderMaxReadTimeSec 
(#31037)"

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.




(beam) branch master updated: Add ScaleToGaussian TFTransform (#31026)

2024-04-19 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new fd1003034ff Add ScaleToGaussian TFTransform (#31026)
fd1003034ff is described below

commit fd1003034ff1232165f963f9aa8b0a128b7ff11e
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Fri Apr 19 11:30:41 2024 -0400

Add ScaleToGaussian TFTransform (#31026)

* Add ScaleToGaussian TFTransform

* address notes

* formatting
---
 sdks/python/apache_beam/ml/transforms/tft.py  |  37 +++
 sdks/python/apache_beam/ml/transforms/tft_test.py | 126 ++
 2 files changed, 163 insertions(+)

diff --git a/sdks/python/apache_beam/ml/transforms/tft.py 
b/sdks/python/apache_beam/ml/transforms/tft.py
index cf44ee9ca0f..9b02cf8b75c 100644
--- a/sdks/python/apache_beam/ml/transforms/tft.py
+++ b/sdks/python/apache_beam/ml/transforms/tft.py
@@ -291,6 +291,43 @@ class ScaleTo01(TFTOperation):
 return output_dict
 
 
+@register_input_dtype(float)
+class ScaleToGaussian(TFTOperation):
+  def __init__(
+  self,
+  columns: List[str],
+  elementwise: bool = False,
+  name: Optional[str] = None):
+"""
+This operation scales the given input column values to an approximately 
+normal distribution with mean 0 and variance of 1. The Gaussian
+transformation is only applied if the column has long tails;
+otherwise, the transformation is the same as normalizing to z scores.
+
+For more information, see: 
+
https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_to_gaussian
+
+Args:
+  columns: A list of column names to apply the transformation on.
+  elementwise: If True, the transformation is applied elementwise.
+Otherwise, the transformation is applied on the entire column.
+  name: A name for the operation (optional).
+
+"""
+super().__init__(columns)
+self.elementwise = elementwise
+self.name = name
+
+  def apply_transform(
+  self, data: common_types.TensorType,
+  output_column_name: str) -> Dict[str, common_types.TensorType]:
+output_dict = {
+output_column_name: tft.scale_to_gaussian(
+x=data, elementwise=self.elementwise, name=self.name)
+}
+return output_dict
+
+
 @register_input_dtype(float)
 class ApplyBuckets(TFTOperation):
   def __init__(
diff --git a/sdks/python/apache_beam/ml/transforms/tft_test.py 
b/sdks/python/apache_beam/ml/transforms/tft_test.py
index 558b4ede2ec..ed7c301f3b8 100644
--- a/sdks/python/apache_beam/ml/transforms/tft_test.py
+++ b/sdks/python/apache_beam/ml/transforms/tft_test.py
@@ -155,6 +155,132 @@ class ScaleTo01Test(unittest.TestCase):
   actual_output, equal_to(expected_output, equals_fn=np.array_equal))
 
 
+class ScaleToGaussianTest(unittest.TestCase):
+  def setUp(self) -> None:
+self.artifact_location = tempfile.mkdtemp()
+
+  def tearDown(self):
+shutil.rmtree(self.artifact_location)
+
+  def test_gaussian_list_uniform_distribution(self):
+list_data = [{'x': [1, 2, 3]}, {'x': [4, 5, 6]}]
+with beam.Pipeline() as p:
+  list_result = (
+  p
+  | "listCreate" >> beam.Create(list_data)
+  | "listMLTransform" >> base.MLTransform(
+  write_artifact_location=self.artifact_location).with_transform(
+  tft.ScaleToGaussian(columns=['x'])))
+
+  expected_data = [
+  np.array([-1.46385, -0.87831, -0.29277], dtype=np.float32),
+  np.array([0.29277, 0.87831, 1.46385], dtype=np.float32)
+  ]
+  actual_data = (list_result | beam.Map(lambda x: x.x))
+  assert_that(
+  actual_data, equal_to(expected_data, equals_fn=np.array_equal))
+
+  def test_gaussian_list_skewed_distribution(self):
+list_data = [{'x': [1, 2, 4]}, {'x': [8, 16, 32]}]
+with beam.Pipeline() as p:
+  list_result = (
+  p
+  | "listCreate" >> beam.Create(list_data)
+  | "listMLTransform" >> base.MLTransform(
+  write_artifact_location=self.artifact_location).with_transform(
+  tft.ScaleToGaussian(columns=['x'])))
+
+  expected_data = [
+  np.array([-0.87733847, -0.78498703, -0.6002842], dtype=np.float32),
+  np.array([-0.23087855, 0.5079328, 1.985], dtype=np.float32)
+  ]
+  actual_data = (list_result | beam.Map(lambda x: x.x))
+  assert_that(actual_data, equal_to(expected_data, equals_fn=np.allclose))
+
+  def test_gaussian_uniform(self):
+data = [
+{
+'x': 1
+},
+{
+'x': 2
+},
+{
+'x': 3
+},
+{
+'x': 4
+},
+{
+   

(beam) branch master updated (b69e8c615af -> 76c77cd28ae)

2024-04-18 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


from b69e8c615af Updates Python Dev container used by Dataflow (#31029)
 add 76c77cd28ae Fix typo in count_unique_words() (#31023)

No new revisions were added by this update.

Summary of changes:
 sdks/python/apache_beam/ml/transforms/tft.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)



(beam) branch master updated: Change type for UnboundedReaderMaxReadTimeSec (#31037)

2024-04-18 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new bb310e7e907 Change type for UnboundedReaderMaxReadTimeSec (#31037)
bb310e7e907 is described below

commit bb310e7e90720b620f1089574f1656ca84a3656d
Author: Radosław Stankiewicz 
AuthorDate: Thu Apr 18 21:50:05 2024 +0200

Change type for UnboundedReaderMaxReadTimeSec (#31037)

* add ms part for UnboundedReader checkpointing

* typo

* spotless

* spotless

* spotless

* [IntLongMath] Expression of type int may overflow before being assigned 
to a long

* readerMaxReadTime sec as double

* readerMaxReadTime sec as double

* readerMaxReadTime sec as double

* readerMaxReadTime sec as double

* spotless
---
 .../runners/dataflow/options/DataflowPipelineDebugOptions.java   | 9 +
 .../apache/beam/runners/dataflow/worker/WorkerCustomSources.java | 4 +++-
 .../beam/runners/dataflow/worker/WorkerCustomSourcesTest.java| 7 ---
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
index 30496dec296..3f6c47ece68 100644
--- 
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
+++ 
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
@@ -218,11 +218,12 @@ public interface DataflowPipelineDebugOptions
 
   /** The max amount of time an UnboundedReader is consumed before 
checkpointing. */
   @Description(
-  "The max amount of time before an UnboundedReader is consumed before 
checkpointing, in seconds.")
-  @Default.Integer(10)
-  Integer getUnboundedReaderMaxReadTimeSec();
+  "The max amount of time before an UnboundedReader is consumed before 
checkpointing, "
+  + "in seconds. Duration can be set to fractions of seconds. ")
+  @Default.Double(10.0)
+  double getUnboundedReaderMaxReadTimeSec();
 
-  void setUnboundedReaderMaxReadTimeSec(Integer value);
+  void setUnboundedReaderMaxReadTimeSec(double value);
 
   /** The max elements read from an UnboundedReader before checkpointing. */
   @Description("The max elements read from an UnboundedReader before 
checkpointing. ")
diff --git 
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
 
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
index 8c086016ee9..a8e358f19e0 100644
--- 
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
+++ 
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
@@ -798,7 +798,9 @@ public class WorkerCustomSources {
   DataflowPipelineDebugOptions debugOptions = 
options.as(DataflowPipelineDebugOptions.class);
   this.endTime =
   Instant.now()
-  
.plus(Duration.standardSeconds(debugOptions.getUnboundedReaderMaxReadTimeSec()));
+  .plus(
+  Duration.millis(
+  (long) (debugOptions.getUnboundedReaderMaxReadTimeSec() 
* 1000L)));
   this.maxElems = debugOptions.getUnboundedReaderMaxElements();
   this.backoffFactory =
   FluentBackoff.DEFAULT
diff --git 
a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
 
b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
index d451ec093f7..261567930fe 100644
--- 
a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
+++ 
b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
@@ -598,6 +598,7 @@ public class WorkerCustomSourcesTest {
 int maxElements = 10;
 DataflowPipelineDebugOptions debugOptions = 
options.as(DataflowPipelineDebugOptions.class);
 debugOptions.setUnboundedReaderMaxElements(maxElements);
+debugOptions.setUnboundedReaderMaxReadTimeSec(10);
 
 ByteString state = ByteString.EMPTY;
 for (int i = 0; i < 10 * maxElements;
@@ -645,10 +646,10 @@ public class WorkerCustomSourcesTest {
 numReadOnThisIteration++;
   }
   Instant afterReading = Instant.now();
-  long maxReadSec = de

(beam) branch master updated: Bump transformers (#30922)

2024-04-12 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new 1a0591ea797 Bump transformers (#30922)
1a0591ea797 is described below

commit 1a0591ea797440d2f4a6eb61fa7d87432856f872
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
AuthorDate: Fri Apr 12 13:53:56 2024 -0400

Bump transformers (#30922)

Bumps [transformers](https://github.com/huggingface/transformers) from 
4.30.0 to 4.38.0.
- [Release notes](https://github.com/huggingface/transformers/releases)
- 
[Commits](https://github.com/huggingface/transformers/compare/v4.30.0...v4.38.0)

---
updated-dependencies:
- dependency-name: transformers
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] 
<49699333+dependabot[bot]@users.noreply.github.com>
---
 .../examples/inference/online_clustering/clustering_pipeline/setup.py   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/sdks/python/apache_beam/examples/inference/online_clustering/clustering_pipeline/setup.py
 
b/sdks/python/apache_beam/examples/inference/online_clustering/clustering_pipeline/setup.py
index 321128cce89..572763492f4 100644
--- 
a/sdks/python/apache_beam/examples/inference/online_clustering/clustering_pipeline/setup.py
+++ 
b/sdks/python/apache_beam/examples/inference/online_clustering/clustering_pipeline/setup.py
@@ -28,7 +28,7 @@ from setuptools import find_packages
 
 REQUIREMENTS = [
 "apache-beam[gcp]==2.40.0",
-"transformers==4.30.0",
+"transformers==4.38.0",
 "torch==1.13.1",
 "scikit-learn==1.0.2",
 ]



(beam) branch master updated: Bump github.com/aws/smithy-go from 1.20.1 to 1.20.2 in /sdks (#30883)

2024-04-08 Thread jrmccluskey
This is an automated email from the ASF dual-hosted git repository.

jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
 new f39a179fbf3 Bump github.com/aws/smithy-go from 1.20.1 to 1.20.2 in 
/sdks (#30883)
f39a179fbf3 is described below

commit f39a179fbf32dd57f425e73027fcb595a6595572
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
AuthorDate: Mon Apr 8 17:25:04 2024 -0400

Bump github.com/aws/smithy-go from 1.20.1 to 1.20.2 in /sdks (#30883)

Bumps [github.com/aws/smithy-go](https://github.com/aws/smithy-go) from 
1.20.1 to 1.20.2.
- [Release notes](https://github.com/aws/smithy-go/releases)
- [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/aws/smithy-go/compare/v1.20.1...v1.20.2)

---
updated-dependencies:
- dependency-name: github.com/aws/smithy-go
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] 
<49699333+dependabot[bot]@users.noreply.github.com>
---
 sdks/go.mod | 2 +-
 sdks/go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/sdks/go.mod b/sdks/go.mod
index 05565b4df47..65d244b265c 100644
--- a/sdks/go.mod
+++ b/sdks/go.mod
@@ -35,7 +35,7 @@ require (
github.com/aws/aws-sdk-go-v2/credentials v1.17.8
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.8
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2
-   github.com/aws/smithy-go v1.20.1
+   github.com/aws/smithy-go v1.20.2
github.com/docker/go-connections v0.5.0
github.com/dustin/go-humanize v1.0.1
github.com/go-sql-driver/mysql v1.8.0
diff --git a/sdks/go.sum b/sdks/go.sum
index d15eeac8589..4a51032d86b 100644
--- a/sdks/go.sum
+++ b/sdks/go.sum
@@ -128,8 +128,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.6.0/go.mod 
h1:q7o0j7d7HrJk/vr9uUt3BV
 github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 
h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg=
 github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod 
h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0=
 github.com/aws/smithy-go v1.6.0/go.mod 
h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
-github.com/aws/smithy-go v1.20.1 
h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw=
-github.com/aws/smithy-go v1.20.1/go.mod 
h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
+github.com/aws/smithy-go v1.20.2 
h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
+github.com/aws/smithy-go v1.20.2/go.mod 
h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/cenkalti/backoff/v4 v4.2.1 
h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod 
h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod 
h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=



  1   2   3   4   5   6   >