svn commit: r70621 [1/2] - /dev/beam/2.58.0/python/
Author: jrmccluskey Date: Tue Jul 30 02:25:02 2024 New Revision: 70621 Log: Staging Python artifacts for Apache Beam 2.58.0 RC Modified: dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-macosx_10_9_x86_64.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-macosx_10_9_x86_64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-macosx_10_9_x86_64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win32.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win32.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win32.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win_amd64.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win_amd64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win_amd64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-macosx_10_9_x86_64.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-macosx_10_9_x86_64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-macosx_10_9_x86_64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win32.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win32.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win32.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win_amd64.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win_amd64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win_amd64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-macosx_10_9_x86_64.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-macosx_10_9_x86_64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-macosx_10_9_x86_64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-win32.whl dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-win32.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-win32.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0
svn commit: r70621 [2/2] - /dev/beam/2.58.0/python/
Modified: dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512 == --- dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512 (original) +++ dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512 Tue Jul 30 02:25:02 2024 @@ -1 +1 @@ -36d5f59aecfd5ae9726f76718b3a2e192a09b15542a2e8c878bc437ded0f620ad91ee53a03eaa83561c7b62f953b5ea10e25d4df0224f1347181b552ed419384 apache_beam-2.58.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl +8f702febb1f103e21f2a811786092571c86435cf3b2a14b4e4ba40b29da8baea6a7a40ff84294fd38489f6a090c566ab1953385abb9b352a3857e51dfbb780a0 apache_beam-2.58.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl Modified: dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl == Binary files - no diff available. Modified: dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl.asc == --- dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl.asc (original) +++ dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl.asc Tue Jul 30 02:25:02 2024 @@ -1,17 +1,17 @@ -BEGIN PGP SIGNATURE- -iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaYHgAYHHByaXZhdGVA -YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiXMoP+gPtVF2oaXOyaUIp6I0JH9c5 -QJdNL9hy4cF/lqamlgSZMaG9++xDArghkoNEL79MBISQF+5A95dNlzzKWw+3EP0L -98GoeooPxfXInXncG119HI25cff+TzBEnaZ+ql5lVnyZgr9fx6z68XaUUcbcnljJ -asc9uFCHm6AsG2NEpCo8ZfQQjgE9y6Xx7dTtCKgff6CZXdgQodFNWHGWXkgG1AC6 -pxwKTfauU9AMelKxIc4S5gFEPXFulO3jJWC5SwAQeBd70XYHhEOevZ0RElb7WoD8 -bnxYwmSHMbJ0b/KYyRHQgMRm5wye9ZqMo9O5aqnEstNQxu8+1pFXqT6a34CMgmqk -mKE5Cfch4DUCGKZKmnOFsIPDOcTVOIDXP4i9HxgZhPW2TVnI9QQHLskFGlVtUFrj -OFPo5ZeTcUlHXhXDqDrJB0QtwrD2gsT+etLvKiRvB0GoTOL/DeI6PUDLmyCCUDfG -cGJdWarXUbHPPlWxf/Ae1ZQqUjUZWjRvFhqZbzkAYgeDcLdgx1jSejFMdttkxzVS -cRQ6ilQQ1QbAVmYvopBJGCuXs5w7nxK9SJinVMpPvv/GIwIhnKsA/iVvFMK8yKZ5 -tBeXEW7G+OlwRtJc3ZtO7QamrxzWFwuLu9hSyvkdxkRlGRH3LlKVZUPcVHwzW98p -V/5grIZ1E5r1dcCdt86R -=vBce +iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaoTuoYHHByaXZhdGVA +YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQicbIQAInIlKXSPvWIUY6d9F8ThH4w +XqGYCwlOPsHNgYsDUb4+ho1TEQI4xJN26DZ837j4L7vvR0s+aS+8e4DLZYpYJHek +cCzNL7TcC3E1d6l8rx59/LSM0qGGsKQmrBaQfFbmni1qA4Ifyp9JBeIqu396pS24 +IBNPnkAR3J2snBZTJTn/AcWf1Ax3qs7ToXi2WZZhMLwHcw2Ye0HxnW4vA7BpqXFN +YJBM39VKNBnhfkfLiSQZUeQk0h3JyPda9tPfV1Y7D7FaG3G3clmwwjNfRUk8qD9p +a3uvky5K54Q11dyFtfuca+zH4ROK+5JopGWQdzO8gVJkrvaDoNA7be7TsITDwaQL +g8/l7Nh2uOIotCqzC1biuou5tZQjoRg429/NJA5Ce4bRpbr7xPZsipC/fMuE7PDh +PcI7WnNtvdEmQnxz9PtZ2KVtGF+BqvqVQvUXLc0UeM67oDyQ77dsg5YHhKd6rDf+ +1hM9LqJjuQ/xZSfBbJfZNZAQjf/g9ygeoVDAIXbe0ZqBM0LjwMHdxqPatcS3CD+J +l+mh1Ys8+AQ3BBXtDE8Zn/gHDpvJG09IiNWwbdm3gf4OLz68DI8APZx5dPFl8pQo +VyQ9KglbzRgrM6wqfGebMajR9ViTMy//vF5/41K2IF03dUWUwVWbwhImpO4GjliS +nsLHiGg/Iy3woC8VTRGz +=PsTY -END PGP SIGNATURE- Modified: dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl.sha512 == --- dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl.sha512 (original) +++ dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl.sha512 Tue Jul 30 02:25:02 2024 @@ -1 +1 @@ -d0af1d34061a25aace0efc544daf78570d537c164cde46349ca22ba6a61358e996c01e70f7795159d7ecfd348de56fd78c387e665bed6a05e79e88daf705923c apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl +4d12ec7f155dc7144721ae383704a933a8c951cb634b2f72d2738c4177c1a4af21989b3bd805bcebf1624dc6b41aab458f879ae2224eba8624cc7c6bed234ae1 apache_beam-2.58.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl Modified: dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl == Binary files - no diff available. Modified: dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc == --- dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc (original) +++ dev/beam/2.58.0/python/apache_beam-2.58.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc Tue Jul 30 02:25:02 2024 @@ -1,17 +1,17 @@ -BEGIN PGP SIGNATURE- -iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaYHgAYHHByaXZhdGVA -YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQivLMP/0eN2lbK3rdNbBP2thoulGEg
svn commit: r70619 - in /dev/beam/2.58.0: apache-beam-2.58.0-source-release.zip apache-beam-2.58.0-source-release.zip.asc apache-beam-2.58.0-source-release.zip.sha512
Author: jrmccluskey Date: Mon Jul 29 21:53:35 2024 New Revision: 70619 Log: Staging Java artifacts for Apache Beam 2.58.0 RC2 Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip (with props) Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip -- svn:mime-type = application/octet-stream Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc == --- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc (original) +++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc Mon Jul 29 21:53:35 2024 @@ -1,17 +1,17 @@ -BEGIN PGP SIGNATURE- -iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaibOMYHHByaXZhdGVA -YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiSh4QAIYodZ8Jh0NdXBclENiPut8p -srQmrZoRKvBOYIBZTl9NzKNVasLRccb3jFRHPnqwSK7GMDKOoijocLpb4ZzxZk4W -vasQfugu33l28aN7Vpx5qcHJrsoNU8md2pgcI7XiPaKocHn/sQsZtzfrfLZUA6iA -/tcZV73kfjKFem/UEeYJbKFfg2GOQjUM8HpUnlV5Jr02FZt2jSnsFApbPZ3RxgSj -5ORE/nH1EiatM71NcHRiRlfY53rub+v6cNAiqgz5SGcFGRX6LbWx7GrnIaMMgtKN -rnaiSfk7ZdjJjiogRIuavdPzvEwCv83odgJd4wi+f+HNPteqJJoisGvnyObj5fs9 -FfI4LXWBcSq2bxjIV5ueKZIQSCbXQIm366E/7P8940V47NSAYRW6NPqsTaY1RQ7c -Ij/Jcbm7oCQQ0xIcQ34RMbAWIee5B9aedGZR60bqqeuzk37e3bkW9yU8uvPUJxW/ -5mx1ockcZNGCecZXzN9IP+l9swOkb23c9k8sUKV1p9dXqBCxTRyZe6z88iq0+Ifd -kBRJ74pMv4Xp/dHemtBx3oAybcBhkdV8k8IdM/OCxIrYXtT9zpuvoS2AoY1aJoVD -aAmiE5N6etD8aRd7DT1seclZPTmeVXHnus4cacMwQhAalTu5OSn4pmcNFVg3Aed3 -M4XbjSl3AX243AVykCPX -=olUN +iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaoD1wYHHByaXZhdGVA +YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQixd4P/1jBZ3Sf5zGvvkXZbvkWVZyk +Nw0pIkjXXCIrb6JoZYyIisCHbqF9fpTxF0IDCw2JhMT/0GewdYIFkY9jO4SWtN4j +qAg+rUcOnEHBjOH5Kekt+EoI5CfJzqqxkO9RNsqmEigLSeIf+le8aEVKaswGcgtq +wCjwJv147HajM48/w0OnknP36oUoA6yVO0BLOZl0rF/dX/ucBm1LhRoWymKBAOim +SeuOaf3WIpPEqV0O8wsKT+OvmfoEpRh5KApAgHWyMINdFE1bCjxZxkCArJ8rPmSS +7sKlmHiJ9ehZmXW/tdDygI4sP0s0I1jqdjFwUB01w91wjBaxe0EzzVwihRlNOggT +8Gvt78svzwkvHNXKxyIY27coHFLggimNitxEUXKk8tCjIOulw3nJKEe/WAptnqau +/o0sxcLRfyczEIcH6AG/wa29sCrYvg4i4nL9aPItqQaqC3v52GrML7G17zPqoejk +VEapnyAJXAYZdGS2O5GUy3u+JTLDFBE4SFTntCQP5IQM9/keFAa3x/wRbjcKVMco +P2/CiNpwh2jE7RrWirz6DKpT7qh/p2aByIymOcDywit0vecR5tCcvzWvAvt3WU24 +QPAh5OzovQW9Q6O2Xriye+XTld+Ig038nqaj6K7L2UIi9sxoPWAMPPeg2kEH3kPz +CsHbHsCx7rn4Ws5fxttb +=mW3A -END PGP SIGNATURE- Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 == --- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 (original) +++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 Mon Jul 29 21:53:35 2024 @@ -1 +1 @@ -2a7ecfe76bebf27555b9357a8b6cc22e5b6fe0e68229028b9e3a8cfc3480e3609d19c04c686b601630e9b08c6a0094af60313076588d454e79dd9e907045c085 apache-beam-2.58.0-source-release.zip +01a37fe442e3f75617aee2b9e91006849e5e497e2fdddc3b14ed4874b1662018dffa9caa548f04458fc23b86ee04219e975ab579e5d0e9098ba1045d97c680e7 apache-beam-2.58.0-source-release.zip
(beam) annotated tag v2.58.0-RC2 updated (d8315f63270 -> 5867feee6a5)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag v2.58.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag v2.58.0-RC2 was modified! *** from d8315f63270 (commit) to 5867feee6a5 (tag) tagging d8315f63270b6b4d79afd2579b41dba23b999235 (commit) replaces jupyterlab-sidepanel-v3.0.0 by Jack McCluskey on Mon Jul 29 17:45:20 2024 -0400 - Log - v2.58.0-RC2 --- No new revisions were added by this update. Summary of changes:
(beam) annotated tag sdks/v2.58.0-RC2 updated (d8315f63270 -> f1090e66603)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag sdks/v2.58.0-RC2 was modified! *** from d8315f63270 (commit) to f1090e66603 (tag) tagging d8315f63270b6b4d79afd2579b41dba23b999235 (commit) replaces jupyterlab-sidepanel-v3.0.0 by Jack McCluskey on Mon Jul 29 17:45:20 2024 -0400 - Log - Go SDK v2.58.0-RC2 --- No new revisions were added by this update. Summary of changes:
(beam) annotated tag sdks/v2.58.0-RC2 deleted (was 5b9a1016dd3)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag sdks/v2.58.0-RC2 was deleted! *** tag was 5b9a1016dd3 This change permanently discards the following revisions: discard 5782d1fd34f Set version for 2.58.0 RC2
(beam) annotated tag v2.58.0-RC2 deleted (was bbfbcc7d1f3)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag v2.58.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag v2.58.0-RC2 was deleted! *** tag was bbfbcc7d1f3 This change permanently discards the following revisions: discard 5782d1fd34f Set version for 2.58.0 RC2
(beam) branch release-2.58.0 updated: Adds ordered list user state support to fnapi accessor cache. (#31923) (#32013)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.58.0 by this push:
new 7fbbe0bf26a Adds ordered list user state support to fnapi accessor
cache. (#31923) (#32013)
7fbbe0bf26a is described below
commit 7fbbe0bf26ae6bf1112efcd0ec5814b03c7b9f0a
Author: Shunping Huang
AuthorDate: Mon Jul 29 17:43:25 2024 -0400
Adds ordered list user state support to fnapi accessor cache. (#31923)
(#32013)
Co-authored-by: Andrew Crites
---
.../java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java | 8 +---
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git
a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java
b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java
index 93f89301d15..5b304890b35 100644
---
a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java
+++
b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java
@@ -813,14 +813,8 @@ public class FnApiStateAccessor implements
SideInputReader, StateBinder {
private Cache getCacheFor(StateKey stateKey) {
switch (stateKey.getTypeCase()) {
case BAG_USER_STATE:
-for (CacheToken token : cacheTokens.get()) {
- if (!token.hasUserState()) {
-continue;
- }
- return Caches.subCache(processWideCache, token, stateKey);
-}
-break;
case MULTIMAP_KEYS_USER_STATE:
+ case ORDERED_LIST_USER_STATE:
for (CacheToken token : cacheTokens.get()) {
if (!token.hasUserState()) {
continue;
(beam) branch master updated (9e431b49bc7 -> 5d13894975b)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 9e431b49bc7 Correctly chose earliest or latest in pane. (#31979) add 5d13894975b Revert "Update Build Wheels to only build once on RCs" (#32014) No new revisions were added by this update. Summary of changes: .github/workflows/build_wheels.yml | 5 + 1 file changed, 1 insertion(+), 4 deletions(-)
(beam) branch revert-32010-wheelCherrypick updated (5afeaf4e13e -> 65022e988e2)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch revert-32010-wheelCherrypick in repository https://gitbox.apache.org/repos/asf/beam.git from 5afeaf4e13e Revert "Update Build Wheels to only build once on RCs (#32009) (#32010)" add 65022e988e2 Add 8 hour timeout No new revisions were added by this update. Summary of changes: .github/workflows/build_wheels.yml | 1 + 1 file changed, 1 insertion(+)
(beam) branch revert-32009-bigWheels updated (9aabe2e07c1 -> 54971749d83)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch revert-32009-bigWheels in repository https://gitbox.apache.org/repos/asf/beam.git from 9aabe2e07c1 Revert "Update Build Wheels to only build once on RCs (#32009)" add 54971749d83 Add 8 hour timeout No new revisions were added by this update. Summary of changes: .github/workflows/build_wheels.yml | 1 + 1 file changed, 1 insertion(+)
(beam) 01/01: Revert "Update Build Wheels to only build once on RCs (#32009) (#32010)"
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch revert-32010-wheelCherrypick
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 5afeaf4e13e656ae0183969a691c73576c9cf7cf
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Mon Jul 29 16:50:28 2024 -0400
Revert "Update Build Wheels to only build once on RCs (#32009) (#32010)"
This reverts commit 0a4bdc370ae6532151bedf296286be34d1c13473.
---
.github/workflows/build_wheels.yml | 4
1 file changed, 4 deletions(-)
diff --git a/.github/workflows/build_wheels.yml
b/.github/workflows/build_wheels.yml
index 35a21626d2a..dc14359aa88 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -236,7 +236,6 @@ jobs:
arch: aarch64
steps:
- name: Download python source distribution from artifacts
- if: ${{ needs.build_source.outputs.is_rc == 0 }}
# Pinned to v3 because of
https://github.com/actions/download-artifact/issues/249
uses: actions/download-artifact@v3
with:
@@ -260,7 +259,6 @@ jobs:
# note: sync cibuildwheel version with gradle task sdks:python:bdistPy*
steps
run: pip install cibuildwheel==2.17.0 setuptools
- name: Build wheel
- if: ${{ needs.build_source.outputs.is_rc == 0 }}
working-directory: apache-beam-source
env:
CIBW_BUILD: ${{ matrix.os_python.python }}
@@ -273,7 +271,6 @@ jobs:
if: startsWith(matrix.os_python.os, 'macos')
run: brew install coreutils
- name: Add checksums
- if: ${{ needs.build_source.outputs.is_rc == 0 }}
working-directory: apache-beam-source/wheelhouse/
run: |
for file in *.whl; do
@@ -281,7 +278,6 @@ jobs:
done
shell: bash
- name: Upload wheels as artifacts
- if: ${{ needs.build_source.outputs.is_rc == 0 }}
# Pinned to v3 because of
https://github.com/actions/download-artifact/issues/249
uses: actions/upload-artifact@v3
with:
(beam) branch revert-32010-wheelCherrypick created (now 5afeaf4e13e)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch revert-32010-wheelCherrypick in repository https://gitbox.apache.org/repos/asf/beam.git at 5afeaf4e13e Revert "Update Build Wheels to only build once on RCs (#32009) (#32010)" This branch includes the following new commits: new 5afeaf4e13e Revert "Update Build Wheels to only build once on RCs (#32009) (#32010)" The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(beam) branch revert-32009-bigWheels created (now 9aabe2e07c1)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch revert-32009-bigWheels in repository https://gitbox.apache.org/repos/asf/beam.git at 9aabe2e07c1 Revert "Update Build Wheels to only build once on RCs (#32009)" This branch includes the following new commits: new 9aabe2e07c1 Revert "Update Build Wheels to only build once on RCs (#32009)" The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(beam) 01/01: Revert "Update Build Wheels to only build once on RCs (#32009)"
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch revert-32009-bigWheels
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 9aabe2e07c1b562c1edd4a5c1074056d4c924d8f
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Mon Jul 29 16:49:42 2024 -0400
Revert "Update Build Wheels to only build once on RCs (#32009)"
This reverts commit ffae5b5bf0b988a9db6e5a1ea93d6ba75e438d08.
---
.github/workflows/build_wheels.yml | 4
1 file changed, 4 deletions(-)
diff --git a/.github/workflows/build_wheels.yml
b/.github/workflows/build_wheels.yml
index 35a21626d2a..dc14359aa88 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -236,7 +236,6 @@ jobs:
arch: aarch64
steps:
- name: Download python source distribution from artifacts
- if: ${{ needs.build_source.outputs.is_rc == 0 }}
# Pinned to v3 because of
https://github.com/actions/download-artifact/issues/249
uses: actions/download-artifact@v3
with:
@@ -260,7 +259,6 @@ jobs:
# note: sync cibuildwheel version with gradle task sdks:python:bdistPy*
steps
run: pip install cibuildwheel==2.17.0 setuptools
- name: Build wheel
- if: ${{ needs.build_source.outputs.is_rc == 0 }}
working-directory: apache-beam-source
env:
CIBW_BUILD: ${{ matrix.os_python.python }}
@@ -273,7 +271,6 @@ jobs:
if: startsWith(matrix.os_python.os, 'macos')
run: brew install coreutils
- name: Add checksums
- if: ${{ needs.build_source.outputs.is_rc == 0 }}
working-directory: apache-beam-source/wheelhouse/
run: |
for file in *.whl; do
@@ -281,7 +278,6 @@ jobs:
done
shell: bash
- name: Upload wheels as artifacts
- if: ${{ needs.build_source.outputs.is_rc == 0 }}
# Pinned to v3 because of
https://github.com/actions/download-artifact/issues/249
uses: actions/upload-artifact@v3
with:
(beam) branch master updated: Regenerate Dataflow Python client (#31997)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 0a65b4b065b Regenerate Dataflow Python client (#31997)
0a65b4b065b is described below
commit 0a65b4b065b27163d6839f9924844c139c7fdf68
Author: Yi Hu
AuthorDate: Mon Jul 29 13:08:18 2024 -0400
Regenerate Dataflow Python client (#31997)
---
.../clients/dataflow/dataflow_v1b3_client.py | 81 +-
.../clients/dataflow/dataflow_v1b3_messages.py | 1497
2 files changed, 1224 insertions(+), 354 deletions(-)
diff --git
a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
index cc982098797..e42b180bbec 100644
---
a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
+++
b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
@@ -1,8 +1,5 @@
"""Generated client library for dataflow version v1b3."""
# NOTE: This file is autogenerated and should not be edited by hand.
-
-from __future__ import absolute_import
-
from apitools.base.py import base_api
from . import dataflow_v1b3_messages as messages
@@ -17,9 +14,7 @@ class DataflowV1b3(base_api.BaseApiClient):
_PACKAGE = 'dataflow'
_SCOPES = [
'https://www.googleapis.com/auth/cloud-platform',
- 'https://www.googleapis.com/auth/compute',
- 'https://www.googleapis.com/auth/compute.readonly',
- 'https://www.googleapis.com/auth/userinfo.email'
+ 'https://www.googleapis.com/auth/compute'
]
_VERSION = 'v1b3'
_CLIENT_ID = '1042881264118.apps.googleusercontent.com'
@@ -75,7 +70,6 @@ class DataflowV1b3(base_api.BaseApiClient):
self.projects_locations_jobs = self.ProjectsLocationsJobsService(self)
self.projects_locations_snapshots = self.ProjectsLocationsSnapshotsService(
self)
-self.projects_locations_sql = self.ProjectsLocationsSqlService(self)
self.projects_locations_templates = self.ProjectsLocationsTemplatesService(
self)
self.projects_locations = self.ProjectsLocationsService(self)
@@ -254,7 +248,7 @@ class DataflowV1b3(base_api.BaseApiClient):
self._upload_configs = {}
def Aggregated(self, request, global_params=None):
- r"""List the jobs of a project across all regions.
+ r"""List the jobs of a project across all regions. **Note:** This method
doesn't support filtering the list of jobs by name.
Args:
request: (DataflowProjectsJobsAggregatedRequest) input message
@@ -270,7 +264,8 @@ class DataflowV1b3(base_api.BaseApiClient):
method_id='dataflow.projects.jobs.aggregated',
ordered_params=['projectId'],
path_params=['projectId'],
-query_params=['filter', 'location', 'pageSize', 'pageToken', 'view'],
+query_params=
+['filter', 'location', 'name', 'pageSize', 'pageToken', 'view'],
relative_path='v1b3/projects/{projectId}/jobs:aggregated',
request_field='',
request_type_name='DataflowProjectsJobsAggregatedRequest',
@@ -279,7 +274,7 @@ class DataflowV1b3(base_api.BaseApiClient):
)
def Create(self, request, global_params=None):
- r"""Creates a Cloud Dataflow job. To create a job, we recommend using
`projects.locations.jobs.create` with a [regional endpoint]
(https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). Using
`projects.jobs.create` is not recommended, as your job will always start in
`us-central1`.
+ r"""Creates a Cloud Dataflow job. To create a job, we recommend using
`projects.locations.jobs.create` with a [regional endpoint]
(https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). Using
`projects.jobs.create` is not recommended, as your job will always start in
`us-central1`. Do not enter confidential information when you supply string
values using the API.
Args:
request: (DataflowProjectsJobsCreateRequest) input message
@@ -354,7 +349,7 @@ class DataflowV1b3(base_api.BaseApiClient):
)
def List(self, request, global_params=None):
- r"""List the jobs of a project. To list the jobs of a project in a
region, we recommend using `projects.locations.jobs.list` with a [regional
endpoint] (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints).
To list the all jobs across all regions, use `projects.jobs.aggregated`. Using
`projects.jobs.list` is not recommended, as you can only get the list of jobs
that are running in `us-central1`.
+ r"""List the jobs of a project. To list the jobs of a project in a
region, we recommend using `proje
(beam) annotated tag sdks/v2.58.0-RC2 updated (5782d1fd34f -> 5b9a1016dd3)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag sdks/v2.58.0-RC2 was modified! *** from 5782d1fd34f (commit) to 5b9a1016dd3 (tag) tagging 5782d1fd34f53027ccc155a2ca70d27086b629cf (commit) replaces jupyterlab-sidepanel-v3.0.0 by Jack McCluskey on Mon Jul 29 10:29:27 2024 -0400 - Log - Go SDK v2.58.0-RC2 --- No new revisions were added by this update. Summary of changes:
(beam) annotated tag v2.58.0-RC2 updated (5782d1fd34f -> bbfbcc7d1f3)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag v2.58.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag v2.58.0-RC2 was modified! *** from 5782d1fd34f (commit) to bbfbcc7d1f3 (tag) tagging 5782d1fd34f53027ccc155a2ca70d27086b629cf (commit) replaces jupyterlab-sidepanel-v3.0.0 by Jack McCluskey on Mon Jul 29 10:29:27 2024 -0400 - Log - v2.58.0-RC2 --- No new revisions were added by this update. Summary of changes:
(beam) annotated tag sdks/v2.58.0-RC2 deleted (was 6e4316aa3d3)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag sdks/v2.58.0-RC2 was deleted! *** tag was 6e4316aa3d3 This change permanently discards the following revisions: discard 898f723b951 Set version for 2.58.0 RC2
(beam) annotated tag v2.58.0-RC2 deleted (was 5bf106b32d7)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag v2.58.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag v2.58.0-RC2 was deleted! *** tag was 5bf106b32d7 The revisions that were on this annotated tag are still contained in other references; therefore, this change does not discard any commits from the repository.
(beam) branch master updated (7930a1fc88b -> ffae5b5bf0b)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 7930a1fc88b Revert "Avoid publishing string set metrics on the Dataflow legacy runner." (#32002) add ffae5b5bf0b Update Build Wheels to only build once on RCs (#32009) No new revisions were added by this update. Summary of changes: .github/workflows/build_wheels.yml | 4 1 file changed, 4 insertions(+)
(beam-site) branch updates_release_2.58.0 updated (c9292ee7bc -> 055335f97f)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch updates_release_2.58.0 in repository https://gitbox.apache.org/repos/asf/beam-site.git omit c9292ee7bc Update beam-site for release 2.58.0. add 055335f97f Update beam-site for release 2.58.0. This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (c9292ee7bc) \ N -- N -- N refs/heads/updates_release_2.58.0 (055335f97f) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: javadoc/2.58.0/deprecated-list.html| 28 +- .../beam/sdk/expansion/service/package-tree.html | 2 +- .../sql/impl/rel/BeamLogicalConvention.html| 2 +- .../sdk/extensions/sql/impl/rel/package-tree.html | 2 +- .../org/apache/beam/sdk/io/cdap/package-tree.html | 2 +- .../org/apache/beam/sdk/io/fs/package-tree.html| 4 +- .../beam/sdk/io/gcp/bigquery/package-tree.html | 10 +- .../beam/sdk/io/gcp/healthcare/package-tree.html | 4 +- .../spanner/changestreams/model/package-tree.html | 2 +- .../org/apache/beam/sdk/io/package-tree.html | 6 +- .../beam/sdk/io/snowflake/enums/package-tree.html | 2 +- .../org/apache/beam/sdk/options/package-tree.html | 24 +- .../2.58.0/org/apache/beam/sdk/package-tree.html | 2 +- .../beam/sdk/schemas/annotations/package-tree.html | 6 +- .../org/apache/beam/sdk/schemas/package-tree.html | 2 +- .../org/apache/beam/sdk/transforms/DoFn.html | 2 +- .../sdk/transforms/display/HasDisplayData.html | 2 +- .../apache/beam/sdk/transforms/package-tree.html | 48 ++-- .../sdk/transforms/windowing/package-tree.html | 4 +- .../org/apache/beam/sdk/values/package-tree.html | 2 +- javadoc/2.58.0/overview-tree.html | 106 .../_modules/apache_beam/yaml/yaml_transform.html | 4 +- pydoc/2.58.0/apache_beam.dataframe.frames.html | 294 ++--- pydoc/2.58.0/apache_beam.dataframe.io.html | 10 +- pydoc/2.58.0/apache_beam.dataframe.transforms.html | 12 +- pydoc/2.58.0/apache_beam.io.fileio.html| 2 +- ...he_beam.ml.inference.huggingface_inference.html | 16 +- .../apache_beam.ml.inference.onnx_inference.html | 6 +- ...apache_beam.ml.inference.pytorch_inference.html | 16 +- ...che_beam.ml.inference.tensorflow_inference.html | 14 +- ...pache_beam.ml.inference.tensorrt_inference.html | 6 +- ...apache_beam.ml.inference.xgboost_inference.html | 22 +- .../2.58.0/apache_beam.ml.transforms.handlers.html | 8 +- pydoc/2.58.0/apache_beam.ml.transforms.tft.html| 26 +- .../apache_beam.transforms.periodicsequence.html | 2 +- pydoc/2.58.0/apache_beam.typehints.schemas.html| 4 +- pydoc/2.58.0/doctest/output.txt| 28 +- .../2.58.0/doctrees/apache_beam.io.fileio.doctree | Bin 83008 -> 83008 bytes ...beam.ml.inference.huggingface_inference.doctree | Bin 124656 -> 124656 bytes ...apache_beam.ml.inference.onnx_inference.doctree | Bin 31312 -> 31312 bytes ...che_beam.ml.inference.pytorch_inference.doctree | Bin 91725 -> 91725 bytes ..._beam.ml.inference.tensorflow_inference.doctree | Bin 76300 -> 76300 bytes ...he_beam.ml.inference.tensorrt_inference.doctree | Bin 43210 -> 43210 bytes ...che_beam.ml.inference.xgboost_inference.doctree | Bin 118378 -> 118378 bytes .../apache_beam.ml.transforms.handlers.doctree | Bin 29035 -> 29035 bytes .../doctrees/apache_beam.ml.transforms.tft.doctree | Bin 128840 -> 128840 bytes ...apache_beam.transforms.periodicsequence.doctree | Bin 37699 -> 37699 bytes .../doctrees/apache_beam.typehints.schemas.doctree | Bin 223622 -> 223622 bytes pydoc/2.58.0/doctrees/environment.pickle | Bin 13109915 -> 13110178 bytes pydoc/2.58.0/searchindex.js| 2 +- .../classes/coders_js_coders.BsonObjectCoder.html | 10 +- .../coders_js_coders.GeneralObjectCoder.html | 16 +- .../classes/coders_required_coders.BytesCoder.html | 14 +- ...ers_required_coders.FullWindowedValueCoder.html | 18 +- .../coders_required_coders.GlobalWindowCoder.html | 12 +- .../coders_required_coders.InstantCoder.html | 12 +- .../coders_required_coders.IterableCoder.html | 18 +- .../classes/coders_required_coders.KVCoder.html| 18 +- ...coders_required_coders.LengthPrefixedCoder.html
svn commit: r70549 - in /dev/beam/2.58.0/prism: darwin/amd64/ darwin/arm64/ linux/amd64/ linux/arm64/ windows/amd64/ windows/arm64/
Author: jrmccluskey Date: Thu Jul 25 15:22:08 2024 New Revision: 70549 Log: Staging Prism artifacts for Apache Beam 2.58.0 RC Modified: dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512 dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip.asc dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip.sha512 dev/beam/2.58.0/prism/linux/amd64/apache_beam-v2.58.0-prism-linux-amd64.zip dev/beam/2.58.0/prism/linux/amd64/apache_beam-v2.58.0-prism-linux-amd64.zip.asc dev/beam/2.58.0/prism/linux/amd64/apache_beam-v2.58.0-prism-linux-amd64.zip.sha512 dev/beam/2.58.0/prism/linux/arm64/apache_beam-v2.58.0-prism-linux-arm64.zip dev/beam/2.58.0/prism/linux/arm64/apache_beam-v2.58.0-prism-linux-arm64.zip.asc dev/beam/2.58.0/prism/linux/arm64/apache_beam-v2.58.0-prism-linux-arm64.zip.sha512 dev/beam/2.58.0/prism/windows/amd64/apache_beam-v2.58.0-prism-windows-amd64.zip dev/beam/2.58.0/prism/windows/amd64/apache_beam-v2.58.0-prism-windows-amd64.zip.asc dev/beam/2.58.0/prism/windows/amd64/apache_beam-v2.58.0-prism-windows-amd64.zip.sha512 dev/beam/2.58.0/prism/windows/arm64/apache_beam-v2.58.0-prism-windows-arm64.zip dev/beam/2.58.0/prism/windows/arm64/apache_beam-v2.58.0-prism-windows-arm64.zip.asc dev/beam/2.58.0/prism/windows/arm64/apache_beam-v2.58.0-prism-windows-arm64.zip.sha512 Modified: dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip == Binary files - no diff available. Modified: dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc == --- dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc (original) +++ dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc Thu Jul 25 15:22:08 2024 @@ -1,17 +1,17 @@ -BEGIN PGP SIGNATURE- -iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaW4MMYHHByaXZhdGVA -YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiy5oP/RmV0UdqiUOSuoHy7WhrJEle -xfW0HIiXguSAUCxMmVOoVbtuHk8lmBcpXkQqbDvTqoaPZbLqFST3nFpf8hxeAmCW -mjiNyZ5fkYjjLVPR7BftjGNzRBoHsjX0B28hwsgA26O2BT7CJdXtGT31xzg413hK -WFoI3UEAkJV1LiSkePCFKmnoPIN+642GztD+LQ7KB32S/Va33i1RjNNS1Wt5mhSh -Rv5gdOCdXBk73nm5Sfvk9PREE6Q3nJzP9ts4Nng0QVJNaXfz4mcWl4UiGbdKr/8d -pAYaKABse5bxL+MStsjj0ZcshUIHHPN9Xs/qqguzfGSmMFFvw88fHYS4xfg/XuKk -BMCokNFfkUrTq74j02k13CTvvayqjuyu/l42sn7W2r0M+6XoTtivHSruSvJechLc -NfiJ4DrUctLAfk133LiCmP0j76kpk6gq5RdMlRSPq9Dz3deE0VcF0egTH/mfUMOa -tUP1VWru8IO0wmfWX+qNM+KpZ+U28bFYpPb4Vzx2BljSgXtPk8BQmciyjw5TVEQd -M458oh+rHU+T9zx8n+TFkGmodyx0/0khmq8LbrRsNGaPNiZs98QH9TfTe7ahYTht -3QirvJqAKQES1GYGNlZI30dH3v1rJvgo4IJ9TkuFc3pFxUe5gViVpb73LQRQf0CX -V23uewV1pHiXXd5vUDe+ -=Z3/h +iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaibX0YHHByaXZhdGVA +YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQivU0QAKbc+uOGdHH0QgRZwx/S57Ri +YhQVaZ8Nw0FctPUAKOgYCkT4fopwXTPQU1DoCnx6nlFmnYWeHWyr1gDZWyitmuzo +/gCx+RPfWFvsd5LEUOcfurf/JVwRv7dFeFEoSPRZssV+PsHi8bNfQmmOmZ9TGJVt +E355S68xZM7RDVPtHNVOdkN1INMmrcw1zpuuGfIrkBxnbUwPMn/LOG3DH9kmlSzH +IbqSvYx/hj17l/nFPqSCuy+XlM2G0gXyPA7Q1X/kZtxk5T+i8h3ZDt7CpGkXyB/B +mFX01UwCaZe6Od4Z555TOeLvSP+gvZTJjyZKETyy/6LhRL4IuKteBjhgnPeCwr6m +LCP0kW9NCmdJ+PujrEFnBjV8wb32dtMWdhce0GduDiilHLD2wVZJFvDGfnzbrHh0 +jrSSGTcY4QepPEiE5NNNzIwLMH4lyBBNyVzUvt/2pInkd4+gR3dIZZGc5YGtrl3V +B3Q2qftqF9rxxxp8MTK/cHScB59PZDpn/1NiQUzKLMyiouqOmOSORpHrthvVwHn4 +CN2MSnbWPLegN6tE8gOyptYPDzwklXpK4UL7OC4awOYTebct+D3Z3IsQSTReezhF +aD1q2zmg7QL5qrtY6v/j3km0kR5Mw8apTcQHT9LtoyGchUjUzAvIX5jvjyNybzfD +UFrUd38o9fIRIFHE7j4y +=aowS -END PGP SIGNATURE- Modified: dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512 == --- dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512 (original) +++ dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512 Thu Jul 25 15:22:08 2024 @@ -1 +1 @@ -a76dc4a051dbbaee8e85073b56921302dce28f619d91be532d4a3b80183d666a9d53cf0f1d01df2dc962e9f7501775f9af0ac29a9104ab7b03223f239ea0605a apache_beam-v2.58.0-prism-darwin-amd64.zip +8a90eb703f047964da820531318e87c9d90bdaa28e945c54ef5397b52d88e467704d5a29cf048834f266c6c2b16f5e2055ff72ffccce60e297a1f56ba35547cd apache_beam-v2.58.0-prism-darwin-amd64.zip Modified: dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip == Binary files - no diff available. Modified: dev/beam/2.58.0/prism
svn commit: r70548 - in /dev/beam/2.58.0: apache-beam-2.58.0-source-release.zip apache-beam-2.58.0-source-release.zip.asc apache-beam-2.58.0-source-release.zip.sha512
Author: jrmccluskey Date: Thu Jul 25 15:19:03 2024 New Revision: 70548 Log: Staging Java artifacts for Apache Beam 2.58.0 RC2 Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip (with props) Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip -- svn:mime-type = application/octet-stream Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc == --- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc (original) +++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc Thu Jul 25 15:19:03 2024 @@ -1,17 +1,17 @@ -BEGIN PGP SIGNATURE- -iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaW3zMYHHByaXZhdGVA -YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQin6MP/3mX5HNCLLAheR94p5jWsobu -UduIN/nrXKYOcDAuulm9bgOAfQgJh/4P8Ds60uf5vFCYtPGWwR3SFOf3jBFgn71c -A990YYS2uU1vCkgI1hHASHNaViV6vuTUre2qUK5nZDRr5GH66GxIQPsIVy/mKojx -9A7gaAueKTG4EmqF5zYJr23vk6ZKEGVBV8mjpsZ5MNsV0NmKtHuzuTX9o4JPAcVa -8jHfyeRfLnzDhLEeGTa30NbN2ofk4YnDrHgsyBfb3DXyBZd5nGxpPTexAH3ehJG7 -XPasWTCo/024XSQwfEhkAcYSCn0t+XgJe/axq+VPC2E9Sdp2lr+MNudYlmefBUIL -YvMM0AazTuYJajJQCBho7Zx5W1aYY1CFM73Af9N/9Fs9i5KfcPeyKvm9ehDC+CD6 -HOXzejGtPBJ3T44DWz/Im+2c6mktDAXmd6adNTw20uxGDYMhESI08cA00pxwTTGA -NLLk0uAQ+QGB0YaLRH7svbZt41pHSTNyVlZWNrKYYGt5Kce3pWGTkyIJc0MBWpII -pS9UmzMm1sjcogDnGcTWl78VLCQby9kgzKyB6DBAJbh63CAUkR/k8jhXqXVz6Q1p -JChap4jSLn0GKnHtSrE1XDXcTUkd/EVu3ASr8564QdpQbhs3nKk6luwtp8DaEyg3 -HA7cTMaUYVsRu1hSdrQL -=zDx7 +iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaibOMYHHByaXZhdGVA +YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiSh4QAIYodZ8Jh0NdXBclENiPut8p +srQmrZoRKvBOYIBZTl9NzKNVasLRccb3jFRHPnqwSK7GMDKOoijocLpb4ZzxZk4W +vasQfugu33l28aN7Vpx5qcHJrsoNU8md2pgcI7XiPaKocHn/sQsZtzfrfLZUA6iA +/tcZV73kfjKFem/UEeYJbKFfg2GOQjUM8HpUnlV5Jr02FZt2jSnsFApbPZ3RxgSj +5ORE/nH1EiatM71NcHRiRlfY53rub+v6cNAiqgz5SGcFGRX6LbWx7GrnIaMMgtKN +rnaiSfk7ZdjJjiogRIuavdPzvEwCv83odgJd4wi+f+HNPteqJJoisGvnyObj5fs9 +FfI4LXWBcSq2bxjIV5ueKZIQSCbXQIm366E/7P8940V47NSAYRW6NPqsTaY1RQ7c +Ij/Jcbm7oCQQ0xIcQ34RMbAWIee5B9aedGZR60bqqeuzk37e3bkW9yU8uvPUJxW/ +5mx1ockcZNGCecZXzN9IP+l9swOkb23c9k8sUKV1p9dXqBCxTRyZe6z88iq0+Ifd +kBRJ74pMv4Xp/dHemtBx3oAybcBhkdV8k8IdM/OCxIrYXtT9zpuvoS2AoY1aJoVD +aAmiE5N6etD8aRd7DT1seclZPTmeVXHnus4cacMwQhAalTu5OSn4pmcNFVg3Aed3 +M4XbjSl3AX243AVykCPX +=olUN -END PGP SIGNATURE- Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 == --- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 (original) +++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 Thu Jul 25 15:19:03 2024 @@ -1 +1 @@ -48b6f6240337f5ea96c7bc9b44de93d581493ff5a4e78006e1c514810853ba972bc9325073b635b16b32c88fbdda8c4f87cb822e8e03de8ebd4cd5fc4254de4f apache-beam-2.58.0-source-release.zip +2a7ecfe76bebf27555b9357a8b6cc22e5b6fe0e68229028b9e3a8cfc3480e3609d19c04c686b601630e9b08c6a0094af60313076588d454e79dd9e907045c085 apache-beam-2.58.0-source-release.zip
(beam) annotated tag v2.58.0-RC2 updated (898f723b951 -> 5bf106b32d7)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag v2.58.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag v2.58.0-RC2 was modified! *** from 898f723b951 (commit) to 5bf106b32d7 (tag) tagging 898f723b951df341a2df800a47ec6959045df49e (commit) replaces jupyterlab-sidepanel-v3.0.0 by jrmccluskey on Thu Jul 25 15:15:31 2024 + - Log - v2.58.0-RC2 --- No new revisions were added by this update. Summary of changes:
(beam) annotated tag sdks/v2.58.0-RC2 updated (898f723b951 -> 6e4316aa3d3)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC2 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag sdks/v2.58.0-RC2 was modified! *** from 898f723b951 (commit) to 6e4316aa3d3 (tag) tagging 898f723b951df341a2df800a47ec6959045df49e (commit) replaces jupyterlab-sidepanel-v3.0.0 by jrmccluskey on Thu Jul 25 15:15:31 2024 + - Log - Go SDK v2.58.0-RC2 --- No new revisions were added by this update. Summary of changes:
(beam) branch release-2.58.0 updated: [yaml] Fix yaml provider schema validation and merging (#31974) (#31984)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.58.0 by this push:
new cdc19d07da9 [yaml] Fix yaml provider schema validation and merging
(#31974) (#31984)
cdc19d07da9 is described below
commit cdc19d07da928f8bf55bffa672a3a19e7941cf73
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Thu Jul 25 11:11:45 2024 -0400
[yaml] Fix yaml provider schema validation and merging (#31974) (#31984)
Signed-off-by: Jeffrey Kinard
Co-authored-by: Jeff Kinard
---
sdks/python/apache_beam/yaml/pipeline.schema.yaml | 6 --
sdks/python/apache_beam/yaml/yaml_transform.py| 4 ++--
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/sdks/python/apache_beam/yaml/pipeline.schema.yaml
b/sdks/python/apache_beam/yaml/pipeline.schema.yaml
index f68a7306d94..c3937e61131 100644
--- a/sdks/python/apache_beam/yaml/pipeline.schema.yaml
+++ b/sdks/python/apache_beam/yaml/pipeline.schema.yaml
@@ -168,8 +168,10 @@ $defs:
providerOrProviderInclude:
if:
- properties:
-include {}
+ allOf: [
+{ properties: { include: { type: string }}},
+{ required: [ "include" ] }
+ ]
then:
$ref: '#/$defs/providerInclude'
else:
diff --git a/sdks/python/apache_beam/yaml/yaml_transform.py
b/sdks/python/apache_beam/yaml/yaml_transform.py
index c1c509ebde2..ffef9bbcd8f 100644
--- a/sdks/python/apache_beam/yaml/yaml_transform.py
+++ b/sdks/python/apache_beam/yaml/yaml_transform.py
@@ -1064,5 +1064,5 @@ def expand_pipeline(
return YamlTransform(
pipeline_as_composite(pipeline_spec['pipeline']),
yaml_provider.merge_providers(
- pipeline_spec.get('providers', []), providers or
- {})).expand(beam.pvalue.PBegin(pipeline))
+ yaml_provider.parse_providers(pipeline_spec.get('providers', [])),
+ providers or {})).expand(beam.pvalue.PBegin(pipeline))
(beam) branch release-2.58.0 updated: Dedup Lineage and getTableToExtract call in BigQuerySourceBase (#31960) (#31972)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.58.0 by this push:
new 3ec56433f49 Dedup Lineage and getTableToExtract call in
BigQuerySourceBase (#31960) (#31972)
3ec56433f49 is described below
commit 3ec56433f497ce2a9e217343de46e2bb99b252d9
Author: Yi Hu
AuthorDate: Thu Jul 25 10:37:08 2024 -0400
Dedup Lineage and getTableToExtract call in BigQuerySourceBase (#31960)
(#31972)
---
.../java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java | 3 ---
1 file changed, 3 deletions(-)
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
index 38c0c8e43b2..998c82ab8d8 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
@@ -157,9 +157,6 @@ abstract class BigQuerySourceBase extends
BoundedSource {
LOG.info("Extract job produced {} files", res.extractedFiles.size());
if (res.extractedFiles.size() > 0) {
BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
-// emit this table ID as a lineage source
-Lineage.getSources()
-.add(BigQueryHelpers.dataCatalogName(getTableToExtract(bqOptions),
bqOptions));
final String extractDestinationDir =
resolveTempLocation(bqOptions.getTempLocation(),
"BigQueryExtractTemp", stepUuid);
// Match all files in the destination directory to stat them in bulk.
(beam) branch master updated: Use buffered loggers that periodically flush. (#31977)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 496e7124272 Use buffered loggers that periodically flush. (#31977)
496e7124272 is described below
commit 496e7124272543ff08caa437e42d11fab99121d8
Author: tvalentyn
AuthorDate: Thu Jul 25 07:22:02 2024 -0700
Use buffered loggers that periodically flush. (#31977)
---
sdks/python/container/piputil.go | 13 ++---
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/sdks/python/container/piputil.go b/sdks/python/container/piputil.go
index 113bf405416..d6250ad2fdc 100644
--- a/sdks/python/container/piputil.go
+++ b/sdks/python/container/piputil.go
@@ -32,6 +32,7 @@ import (
"github.com/apache/beam/sdks/v2/go/pkg/beam/util/execx"
)
+const pipLogFlushInterval time.Duration = 15 * time.Second
const unrecoverableURL string =
"https://beam.apache.org/documentation/sdks/python-unrecoverable-errors/index.html#pip-dependency-resolution-failures;
// pipInstallRequirements installs the given requirement, if present.
@@ -40,7 +41,7 @@ func pipInstallRequirements(ctx context.Context, logger
*tools.Logger, files []s
if err != nil {
return err
}
- bufLogger := tools.NewBufferedLogger(logger)
+ bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger,
pipLogFlushInterval)
for _, file := range files {
if file == name {
// We run the install process in two rounds in order to
avoid as much
@@ -48,7 +49,7 @@ func pipInstallRequirements(ctx context.Context, logger
*tools.Logger, files []s
// option will make sure that only things staged in the
worker will be
// used without following their dependencies.
args := []string{"-m", "pip", "install", "-r",
filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check",
"--no-index", "--no-deps", "--find-links", dir}
- if err := execx.Execute(pythonVersion, args...); err !=
nil {
+ if err := execx.ExecuteEnvWithIO(nil, os.Stdin,
bufLogger, bufLogger, pythonVersion, args...); err != nil {
bufLogger.Printf(ctx, "Some packages could not
be installed solely from the requirements cache. Installing packages from
PyPI.")
}
// The second install round opens up the search for
packages on PyPI and
@@ -79,8 +80,6 @@ func isPackageInstalled(pkgName string) bool {
return true
}
-const pipLogFlushInterval time.Duration = 15 * time.Second
-
// pipInstallPackage installs the given package, if present.
func pipInstallPackage(ctx context.Context, logger *tools.Logger, files
[]string, dir, name string, force, optional bool, extras []string) error {
pythonVersion, err := expansionx.GetPythonVersion()
@@ -150,7 +149,7 @@ func pipInstallPackage(ctx context.Context, logger
*tools.Logger, files []string
// installExtraPackages installs all the packages declared in the extra
// packages manifest file.
func installExtraPackages(ctx context.Context, logger *tools.Logger, files
[]string, extraPackagesFile, dir string) error {
- bufLogger := tools.NewBufferedLogger(logger)
+ bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger,
pipLogFlushInterval)
// First check that extra packages manifest file is present.
for _, file := range files {
if file != extraPackagesFile {
@@ -179,7 +178,7 @@ func installExtraPackages(ctx context.Context, logger
*tools.Logger, files []str
}
func findBeamSdkWhl(ctx context.Context, logger *tools.Logger, files []string,
acceptableWhlSpecs []string) string {
- bufLogger := tools.NewBufferedLogger(logger)
+ bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger,
pipLogFlushInterval)
for _, file := range files {
if strings.HasPrefix(file, "apache_beam") {
for _, s := range acceptableWhlSpecs {
@@ -200,7 +199,7 @@ func findBeamSdkWhl(ctx context.Context, logger
*tools.Logger, files []string, a
// SDK from source tarball provided in sdkSrcFile.
func installSdk(ctx context.Context, logger *tools.Logger, files []string,
workDir string, sdkSrcFile string, acceptableWhlSpecs []string, required bool)
error {
sdkWhlFile := findBeamSdkWhl(ctx, logger, files, acceptableWhlSpecs)
- bufLogger := tools.NewBufferedLogger(logger)
+ bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger,
pipLogFlushInterval)
if sdkWhlFile != "" {
// by default, pip rejects to install wheel if same version
already installed
isDev := strings.Contains(sdkWhlFile, ".dev")
(beam) branch master updated: Note SpannerIO.read new validation in 2.58.0 breaking change section (#31973)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 8709f1126f6 Note SpannerIO.read new validation in 2.58.0 breaking change section (#31973) 8709f1126f6 is described below commit 8709f1126f6c0975790ce7b5974a02429847de5f Author: Yi Hu AuthorDate: Thu Jul 25 09:48:27 2024 -0400 Note SpannerIO.read new validation in 2.58.0 breaking change section (#31973) --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index c4698e5585e..2bc2ecc4997 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -112,6 +112,7 @@ * X behavior was changed ([#X](https://github.com/apache/beam/issues/X)). * [IcebergIO] IcebergCatalogConfig was changed to support specifying catalog properties in a key-store fashion ([#31726](https://github.com/apache/beam/pull/31726)) +* [SpannerIO] Added validation that query and table cannot be specified at the same time for SpannerIO.read(). Previously withQuery overrides withTable, if set ([#24956](https://github.com/apache/beam/issues/24956)). ## Deprecations
(beam) branch release-2.58.0 updated: Use buffered loggers that periodically flush. (#31978)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.58.0 by this push:
new 6e19e383972 Use buffered loggers that periodically flush. (#31978)
6e19e383972 is described below
commit 6e19e38397286c75847732d5f5cf52f235f03fc5
Author: tvalentyn
AuthorDate: Thu Jul 25 06:46:11 2024 -0700
Use buffered loggers that periodically flush. (#31978)
---
sdks/python/container/piputil.go | 13 ++---
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/sdks/python/container/piputil.go b/sdks/python/container/piputil.go
index 113bf405416..d6250ad2fdc 100644
--- a/sdks/python/container/piputil.go
+++ b/sdks/python/container/piputil.go
@@ -32,6 +32,7 @@ import (
"github.com/apache/beam/sdks/v2/go/pkg/beam/util/execx"
)
+const pipLogFlushInterval time.Duration = 15 * time.Second
const unrecoverableURL string =
"https://beam.apache.org/documentation/sdks/python-unrecoverable-errors/index.html#pip-dependency-resolution-failures;
// pipInstallRequirements installs the given requirement, if present.
@@ -40,7 +41,7 @@ func pipInstallRequirements(ctx context.Context, logger
*tools.Logger, files []s
if err != nil {
return err
}
- bufLogger := tools.NewBufferedLogger(logger)
+ bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger,
pipLogFlushInterval)
for _, file := range files {
if file == name {
// We run the install process in two rounds in order to
avoid as much
@@ -48,7 +49,7 @@ func pipInstallRequirements(ctx context.Context, logger
*tools.Logger, files []s
// option will make sure that only things staged in the
worker will be
// used without following their dependencies.
args := []string{"-m", "pip", "install", "-r",
filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check",
"--no-index", "--no-deps", "--find-links", dir}
- if err := execx.Execute(pythonVersion, args...); err !=
nil {
+ if err := execx.ExecuteEnvWithIO(nil, os.Stdin,
bufLogger, bufLogger, pythonVersion, args...); err != nil {
bufLogger.Printf(ctx, "Some packages could not
be installed solely from the requirements cache. Installing packages from
PyPI.")
}
// The second install round opens up the search for
packages on PyPI and
@@ -79,8 +80,6 @@ func isPackageInstalled(pkgName string) bool {
return true
}
-const pipLogFlushInterval time.Duration = 15 * time.Second
-
// pipInstallPackage installs the given package, if present.
func pipInstallPackage(ctx context.Context, logger *tools.Logger, files
[]string, dir, name string, force, optional bool, extras []string) error {
pythonVersion, err := expansionx.GetPythonVersion()
@@ -150,7 +149,7 @@ func pipInstallPackage(ctx context.Context, logger
*tools.Logger, files []string
// installExtraPackages installs all the packages declared in the extra
// packages manifest file.
func installExtraPackages(ctx context.Context, logger *tools.Logger, files
[]string, extraPackagesFile, dir string) error {
- bufLogger := tools.NewBufferedLogger(logger)
+ bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger,
pipLogFlushInterval)
// First check that extra packages manifest file is present.
for _, file := range files {
if file != extraPackagesFile {
@@ -179,7 +178,7 @@ func installExtraPackages(ctx context.Context, logger
*tools.Logger, files []str
}
func findBeamSdkWhl(ctx context.Context, logger *tools.Logger, files []string,
acceptableWhlSpecs []string) string {
- bufLogger := tools.NewBufferedLogger(logger)
+ bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger,
pipLogFlushInterval)
for _, file := range files {
if strings.HasPrefix(file, "apache_beam") {
for _, s := range acceptableWhlSpecs {
@@ -200,7 +199,7 @@ func findBeamSdkWhl(ctx context.Context, logger
*tools.Logger, files []string, a
// SDK from source tarball provided in sdkSrcFile.
func installSdk(ctx context.Context, logger *tools.Logger, files []string,
workDir string, sdkSrcFile string, acceptableWhlSpecs []string, required bool)
error {
sdkWhlFile := findBeamSdkWhl(ctx, logger, files, acceptableWhlSpecs)
- bufLogger := tools.NewBufferedLogger(logger)
+ bufLogger := tools.NewBufferedLoggerWithFlushInterval(ctx, logger,
pipLogFlushInterval)
if sdkWhlFile != "" {
// by default, pip rejects to install wheel if same version
already installed
isDev := strings.Contains(sdkWhlFile, ".dev")
(beam) branch master updated: Auth with project id in our notebook (#31966)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new bf7f254 Auth with project id in our notebook (#31966)
bf7f254 is described below
commit bf7f2540f35bce23b251672575224e0762c1
Author: Danny McCormick
AuthorDate: Wed Jul 24 17:15:58 2024 +0200
Auth with project id in our notebook (#31966)
* Auth with project id
* Wording/formatting
---
.../beam-ml/bigtable_enrichment_transform.ipynb| 28 --
1 file changed, 10 insertions(+), 18 deletions(-)
diff --git a/examples/notebooks/beam-ml/bigtable_enrichment_transform.ipynb
b/examples/notebooks/beam-ml/bigtable_enrichment_transform.ipynb
index 7510831bac4..95be8b1d957 100644
--- a/examples/notebooks/beam-ml/bigtable_enrichment_transform.ipynb
+++ b/examples/notebooks/beam-ml/bigtable_enrichment_transform.ipynb
@@ -139,41 +139,33 @@
},
"source": [
"### Authenticate with Google Cloud\n",
-"This notebook reads data from Pub/Sub and Bigtable. To use your
Google Cloud account, authenticate this notebook."
+"This notebook reads data from Pub/Sub and Bigtable. To use your
Google Cloud account, authenticate this notebook.\n",
+"To prepare for this step, replace ``, ``,
and `` with the appropriate values for your setup. These fields are
used with Bigtable."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
-"id": "Kz9sccyGBqz3"
+"id": "wEXucyi2liij"
},
"outputs": [],
"source": [
-"from google.colab import auth\n",
-"auth.authenticate_user()"
- ]
-},
-{
- "cell_type": "markdown",
- "metadata": {
-"id": "nAmGgUMt48o9"
- },
- "source": [
-"Replace ``, ``, and `` with the
appropriate values for your setup. These fields are used with Bigtable."
+"PROJECT_ID = \"\"\n",
+"INSTANCE_ID = \"\"\n",
+"TABLE_ID = \"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
-"id": "wEXucyi2liij"
+"id": "Kz9sccyGBqz3"
},
"outputs": [],
"source": [
-"PROJECT_ID = \"\"\n",
-"INSTANCE_ID = \"\"\n",
-"TABLE_ID = \"\""
+"from google.colab import auth\n",
+"auth.authenticate_user(project_id=PROJECT_ID)"
]
},
{
@@ -879,4 +871,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
-}
\ No newline at end of file
+}
(beam) branch master updated (8f2c72e8d22 -> 521586e04a7)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 8f2c72e8d22 Disable Dataflow run for java_test and python_test (#31934) add 521586e04a7 Update Python artifact name in release process (#31933) No new revisions were added by this update. Summary of changes: contributor-docs/release-guide.md | 12 +++ release/src/main/scripts/run_rc_validation.sh | 48 +-- 2 files changed, 30 insertions(+), 30 deletions(-)
svn commit: r70374 - /dev/beam/2.58.0/python/
Author: jrmccluskey Date: Wed Jul 17 19:40:23 2024 New Revision: 70374 Log: Staging Python artifacts for Apache Beam 2.58.0 RC Added: dev/beam/2.58.0/python/ dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-macosx_10_9_x86_64.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-macosx_10_9_x86_64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-macosx_10_9_x86_64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win32.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win32.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win32.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win_amd64.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win_amd64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp310-cp310-win_amd64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-macosx_10_9_x86_64.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-macosx_10_9_x86_64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-macosx_10_9_x86_64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win32.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win32.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win32.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win_amd64.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win_amd64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp311-cp311-win_amd64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-macosx_10_9_x86_64.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-macosx_10_9_x86_64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-macosx_10_9_x86_64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl.sha512 dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (with props) dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.asc dev/beam/2.58.0/python/apache_beam-2.58.0-cp312-cp312-manylinux_2_17_x86_64
(beam) annotated tag v2.58.0-RC1 updated (5b18d936cc0 -> 5ed765fbc2c)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag v2.58.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag v2.58.0-RC1 was modified! *** from 5b18d936cc0 (commit) to 5ed765fbc2c (tag) tagging 5b18d936cc08507c3341c33e0977484b550298d0 (commit) replaces jupyterlab-sidepanel-v3.0.0 by Jack McCluskey on Wed Jul 17 10:40:48 2024 -0400 - Log - v2.58.0-RC1 --- No new revisions were added by this update. Summary of changes:
(beam) annotated tag sdks/v2.58.0-RC1 updated (5b18d936cc0 -> a262768fd65)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag sdks/v2.58.0-RC1 was modified! *** from 5b18d936cc0 (commit) to a262768fd65 (tag) tagging 5b18d936cc08507c3341c33e0977484b550298d0 (commit) replaces jupyterlab-sidepanel-v3.0.0 by Jack McCluskey on Wed Jul 17 10:40:48 2024 -0400 - Log - Go SDK v2.58.0-RC1 --- No new revisions were added by this update. Summary of changes:
(beam) branch release-2.58.0 updated: Switch to use self-hosted runner for build_wheels action (#31866) (#31916)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.58.0 by this push:
new eca36f69269 Switch to use self-hosted runner for build_wheels action
(#31866) (#31916)
eca36f69269 is described below
commit eca36f6926967fae4f803baa4b1358d5faccab73
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Wed Jul 17 10:40:04 2024 -0400
Switch to use self-hosted runner for build_wheels action (#31866) (#31916)
* Switch to use self-hosted runner
* split os and runner string
Co-authored-by: Yi Hu
---
.github/workflows/build_wheels.yml | 52 --
1 file changed, 16 insertions(+), 36 deletions(-)
diff --git a/.github/workflows/build_wheels.yml
b/.github/workflows/build_wheels.yml
index 5142b0b22c3..dc14359aa88 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -45,7 +45,7 @@ jobs:
check_env_variables:
timeout-minutes: 5
name: "Check environment variables"
-runs-on: ubuntu-latest
+runs-on: [self-hosted, ubuntu-20.04, main]
env:
EVENT_NAME: ${{ github.event_name }}
PY_VERSIONS_FULL: "cp38-* cp39-* cp310-* cp311-* cp312-*"
@@ -59,8 +59,8 @@ jobs:
run: "./scripts/ci/ci_check_are_gcp_variables_set.sh"
id: check_gcp_variables
env:
- GCP_SA_EMAIL: ${{ secrets.GCP_SA_EMAIL }}
- GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+ GCP_SA_EMAIL: "not used by self hosted runner"
+ GCP_SA_KEY: "not used by self hosted runner"
GCP_PYTHON_WHEELS_BUCKET: ${{ secrets.GCP_PYTHON_WHEELS_BUCKET }}
GCP_PROJECT_ID: "not-needed-here"
GCP_REGION: "not-needed-here"
@@ -80,7 +80,7 @@ jobs:
echo "py-versions-full=$PY_VERSIONS_FULL" >> $GITHUB_OUTPUT
build_source:
-runs-on: ubuntu-latest
+runs-on: [self-hosted, ubuntu-20.04, main]
name: Build python source distribution
outputs:
is_rc: ${{ steps.is_rc.outputs.is_rc }}
@@ -190,14 +190,9 @@ jobs:
needs:
- build_source
- check_env_variables
-runs-on: ubuntu-latest
+runs-on: [self-hosted, ubuntu-20.04, main]
if: needs.check_env_variables.outputs.gcp-variables-set == 'true' &&
github.event_name != 'pull_request'
steps:
- - name: Authenticate on GCP
-uses: google-github-actions/setup-gcloud@v0
-with:
- service_account_email: ${{ secrets.GCP_SA_EMAIL }}
- service_account_key: ${{ secrets.GCP_SA_KEY }}
- name: Remove existing files on GCS bucket
run: gsutil rm -r ${{ env.GCP_PATH }} || true
@@ -206,7 +201,7 @@ jobs:
needs:
- prepare_gcs
- check_env_variables
-runs-on: ubuntu-latest
+runs-on: [self-hosted, ubuntu-20.04, main]
if: needs.check_env_variables.outputs.gcp-variables-set == 'true'
steps:
- name: Download compressed sources from artifacts
@@ -215,11 +210,6 @@ jobs:
with:
name: source_zip
path: source/
- - name: Authenticate on GCP
-uses: google-github-actions/setup-gcloud@v0
-with:
- service_account_email: ${{ secrets.GCP_SA_EMAIL }}
- service_account_key: ${{ secrets.GCP_SA_KEY }}
- name: Copy sources to GCS bucket
run: gsutil cp -r -a public-read source/* ${{ env.GCP_PATH }}
@@ -230,19 +220,19 @@ jobs:
- build_source
env:
CIBW_ARCHS_LINUX: ${{matrix.arch}}
-runs-on: ${{ matrix.os_python.os }}
+runs-on: ${{ matrix.os_python.runner }}
strategy:
matrix:
os_python: [
- {"os": "ubuntu-latest", "python": "${{
needs.check_env_variables.outputs.py-versions-full }}" },
+ {"os": "ubuntu-20.04", "runner": [self-hosted, ubuntu-20.04, main],
"python": "${{ needs.check_env_variables.outputs.py-versions-full }}" },
# Temporarily pin to macos-13 because macos-latest breaks this build
# TODO(https://github.com/apache/beam/issues/31114)
- {"os": "macos-13", "python": "${{
needs.check_env_variables.outputs.py-versions-test }}" },
- {"os": "windows-latest", "python": "${{
needs.check_env_variables.outputs.py-versions-test }}" },
+ {"os": "macos-13", "runner": "macos-13", "python": "${{
needs.check_env_variables.outputs.py-versions-test }}" },
+ {"os": "windows-latest", "runner": &quo
(beam) annotated tag sdks/v2.58.0-RC1 deleted (was e264a659ecd)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag sdks/v2.58.0-RC1 was deleted! *** tag was e264a659ecd This change permanently discards the following revisions: discard 06ef4974894 Set version for 2.58.0 RC1
(beam) annotated tag v2.58.0-RC1 deleted (was d1da3b2a5db)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag v2.58.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag v2.58.0-RC1 was deleted! *** tag was d1da3b2a5db The revisions that were on this annotated tag are still contained in other references; therefore, this change does not discard any commits from the repository.
(beam-site) branch updates_release_2.58.0 created (now c9292ee7bc)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch updates_release_2.58.0 in repository https://gitbox.apache.org/repos/asf/beam-site.git at c9292ee7bc Update beam-site for release 2.58.0. No new revisions were added by this update.
svn commit: r70354 - in /dev/beam/2.58.0/prism: ./ darwin/ darwin/amd64/ darwin/arm64/ linux/ linux/amd64/ linux/arm64/ windows/ windows/amd64/ windows/arm64/
Author: jrmccluskey Date: Tue Jul 16 21:07:04 2024 New Revision: 70354 Log: Staging Prism artifacts for Apache Beam 2.58.0 RC Added: dev/beam/2.58.0/prism/ dev/beam/2.58.0/prism/darwin/ dev/beam/2.58.0/prism/darwin/amd64/ dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip (with props) dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512 dev/beam/2.58.0/prism/darwin/arm64/ dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip (with props) dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip.asc dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip.sha512 dev/beam/2.58.0/prism/linux/ dev/beam/2.58.0/prism/linux/amd64/ dev/beam/2.58.0/prism/linux/amd64/apache_beam-v2.58.0-prism-linux-amd64.zip (with props) dev/beam/2.58.0/prism/linux/amd64/apache_beam-v2.58.0-prism-linux-amd64.zip.asc dev/beam/2.58.0/prism/linux/amd64/apache_beam-v2.58.0-prism-linux-amd64.zip.sha512 dev/beam/2.58.0/prism/linux/arm64/ dev/beam/2.58.0/prism/linux/arm64/apache_beam-v2.58.0-prism-linux-arm64.zip (with props) dev/beam/2.58.0/prism/linux/arm64/apache_beam-v2.58.0-prism-linux-arm64.zip.asc dev/beam/2.58.0/prism/linux/arm64/apache_beam-v2.58.0-prism-linux-arm64.zip.sha512 dev/beam/2.58.0/prism/windows/ dev/beam/2.58.0/prism/windows/amd64/ dev/beam/2.58.0/prism/windows/amd64/apache_beam-v2.58.0-prism-windows-amd64.zip (with props) dev/beam/2.58.0/prism/windows/amd64/apache_beam-v2.58.0-prism-windows-amd64.zip.asc dev/beam/2.58.0/prism/windows/amd64/apache_beam-v2.58.0-prism-windows-amd64.zip.sha512 dev/beam/2.58.0/prism/windows/arm64/ dev/beam/2.58.0/prism/windows/arm64/apache_beam-v2.58.0-prism-windows-arm64.zip (with props) dev/beam/2.58.0/prism/windows/arm64/apache_beam-v2.58.0-prism-windows-arm64.zip.asc dev/beam/2.58.0/prism/windows/arm64/apache_beam-v2.58.0-prism-windows-arm64.zip.sha512 Added: dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip == Binary file - no diff available. Propchange: dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip -- svn:mime-type = application/octet-stream Added: dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc == --- dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc (added) +++ dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.asc Tue Jul 16 21:07:04 2024 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- + +iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaW4MMYHHByaXZhdGVA +YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiy5oP/RmV0UdqiUOSuoHy7WhrJEle +xfW0HIiXguSAUCxMmVOoVbtuHk8lmBcpXkQqbDvTqoaPZbLqFST3nFpf8hxeAmCW +mjiNyZ5fkYjjLVPR7BftjGNzRBoHsjX0B28hwsgA26O2BT7CJdXtGT31xzg413hK +WFoI3UEAkJV1LiSkePCFKmnoPIN+642GztD+LQ7KB32S/Va33i1RjNNS1Wt5mhSh +Rv5gdOCdXBk73nm5Sfvk9PREE6Q3nJzP9ts4Nng0QVJNaXfz4mcWl4UiGbdKr/8d +pAYaKABse5bxL+MStsjj0ZcshUIHHPN9Xs/qqguzfGSmMFFvw88fHYS4xfg/XuKk +BMCokNFfkUrTq74j02k13CTvvayqjuyu/l42sn7W2r0M+6XoTtivHSruSvJechLc +NfiJ4DrUctLAfk133LiCmP0j76kpk6gq5RdMlRSPq9Dz3deE0VcF0egTH/mfUMOa +tUP1VWru8IO0wmfWX+qNM+KpZ+U28bFYpPb4Vzx2BljSgXtPk8BQmciyjw5TVEQd +M458oh+rHU+T9zx8n+TFkGmodyx0/0khmq8LbrRsNGaPNiZs98QH9TfTe7ahYTht +3QirvJqAKQES1GYGNlZI30dH3v1rJvgo4IJ9TkuFc3pFxUe5gViVpb73LQRQf0CX +V23uewV1pHiXXd5vUDe+ +=Z3/h +-END PGP SIGNATURE- Added: dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512 == --- dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512 (added) +++ dev/beam/2.58.0/prism/darwin/amd64/apache_beam-v2.58.0-prism-darwin-amd64.zip.sha512 Tue Jul 16 21:07:04 2024 @@ -0,0 +1 @@ +a76dc4a051dbbaee8e85073b56921302dce28f619d91be532d4a3b80183d666a9d53cf0f1d01df2dc962e9f7501775f9af0ac29a9104ab7b03223f239ea0605a apache_beam-v2.58.0-prism-darwin-amd64.zip Added: dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip == Binary file - no diff available. Propchange: dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip -- svn:mime-type = application/octet-stream Added: dev/beam/2.58.0/prism/darwin/arm64/apache_beam-v2.58.0-prism-darwin-arm64.zip.asc
svn commit: r70353 - in /dev/beam/2.58.0: apache-beam-2.58.0-source-release.zip apache-beam-2.58.0-source-release.zip.asc apache-beam-2.58.0-source-release.zip.sha512
Author: jrmccluskey Date: Tue Jul 16 20:59:39 2024 New Revision: 70353 Log: Staging Java artifacts for Apache Beam 2.58.0 RC1 Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip (with props) Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip -- svn:mime-type = application/octet-stream Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc == --- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc (original) +++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc Tue Jul 16 20:59:39 2024 @@ -1,17 +1,17 @@ -BEGIN PGP SIGNATURE- -iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaWv9QYHHByaXZhdGVA -YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiQyIQAMHFAKjHh66owHM98d8tLZXv -x6JpzyILoWu36GahKOXBT1pCqyXguoKm01L/EpCBYFEcoICY8q4KaVgoQZvefRDe -/8c7pkaVXX25lbnfiKdzur2sXi6dsN8LP3P41yo9y7V6x+moWyMr/GcnsWPUBs6y -LDnp7DL2E+EhFpWyO4Iey+KG3KH8jttbGKTgGPcuh3aYFjsGpQRKsHsgYXIR3fAP -e4kK8dp3AZ26oHUIaLfH9HyxqLUB1HxqYViealy4AnIS6Q6AuoPhBi4oeFegyl+m -ZUNEX2eWXf9gA9G77SzeS3UF54NN+8SRbPabG7wvnRiyUGAYwcvug+bnhtYhrD02 -CRePK0ZLLCUyYVojVofvTLV/3fu5Qv+5jWudn48wWH2bNJn3fhbmFuf4qHm4OXgW -bWUmSbslQXuqmVFZbDXYYwplrG1cH55rR/ydwSoobvfiybUPu+O9ltPgQLTS1Hxh -xDW1zKnioXJm2UZik7f+62c5B+pfIWQ9mDPN4h+Jlj9PJDM1KWfCRx9NoWsvKlkc -g5Oo0xul+qF4+ER78h/+0bKhVNbkdo2nGsxPB9Di1LIyzkcLs5H8vHJB/un6IZjl -1gT4UDNdfr6ZHKb8MKMrAFyEx0kuXsDHXa405gZgRNNNQGbq2kUcFZP6thIERlgS -gvLNAdj8pQbw+bzRNdAR -=bT/s +iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaW3zMYHHByaXZhdGVA +YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQin6MP/3mX5HNCLLAheR94p5jWsobu +UduIN/nrXKYOcDAuulm9bgOAfQgJh/4P8Ds60uf5vFCYtPGWwR3SFOf3jBFgn71c +A990YYS2uU1vCkgI1hHASHNaViV6vuTUre2qUK5nZDRr5GH66GxIQPsIVy/mKojx +9A7gaAueKTG4EmqF5zYJr23vk6ZKEGVBV8mjpsZ5MNsV0NmKtHuzuTX9o4JPAcVa +8jHfyeRfLnzDhLEeGTa30NbN2ofk4YnDrHgsyBfb3DXyBZd5nGxpPTexAH3ehJG7 +XPasWTCo/024XSQwfEhkAcYSCn0t+XgJe/axq+VPC2E9Sdp2lr+MNudYlmefBUIL +YvMM0AazTuYJajJQCBho7Zx5W1aYY1CFM73Af9N/9Fs9i5KfcPeyKvm9ehDC+CD6 +HOXzejGtPBJ3T44DWz/Im+2c6mktDAXmd6adNTw20uxGDYMhESI08cA00pxwTTGA +NLLk0uAQ+QGB0YaLRH7svbZt41pHSTNyVlZWNrKYYGt5Kce3pWGTkyIJc0MBWpII +pS9UmzMm1sjcogDnGcTWl78VLCQby9kgzKyB6DBAJbh63CAUkR/k8jhXqXVz6Q1p +JChap4jSLn0GKnHtSrE1XDXcTUkd/EVu3ASr8564QdpQbhs3nKk6luwtp8DaEyg3 +HA7cTMaUYVsRu1hSdrQL +=zDx7 -END PGP SIGNATURE- Modified: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 == --- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 (original) +++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 Tue Jul 16 20:59:39 2024 @@ -1 +1 @@ -845a828fefeee560fb6b2d9bc318cc4a7c4682e1410053f7b5a03064becba40a6d585eb453b1a2af0a04563b95ae441d59c317ec255f86dc65ad7e191b9545af apache-beam-2.58.0-source-release.zip +48b6f6240337f5ea96c7bc9b44de93d581493ff5a4e78006e1c514810853ba972bc9325073b635b16b32c88fbdda8c4f87cb822e8e03de8ebd4cd5fc4254de4f apache-beam-2.58.0-source-release.zip
(beam) annotated tag v2.58.0-RC1 updated (06ef4974894 -> d1da3b2a5db)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag v2.58.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag v2.58.0-RC1 was modified! *** from 06ef4974894 (commit) to d1da3b2a5db (tag) tagging 06ef49748946e9bfef6fe28332089a39aff7fbe7 (commit) replaces jupyterlab-sidepanel-v3.0.0 by Jack McCluskey on Tue Jul 16 16:51:06 2024 -0400 - Log - v2.58.0-RC1 --- No new revisions were added by this update. Summary of changes:
(beam) annotated tag sdks/v2.58.0-RC1 updated (06ef4974894 -> e264a659ecd)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag sdks/v2.58.0-RC1 was modified! *** from 06ef4974894 (commit) to e264a659ecd (tag) tagging 06ef49748946e9bfef6fe28332089a39aff7fbe7 (commit) replaces jupyterlab-sidepanel-v3.0.0 by Jack McCluskey on Tue Jul 16 16:51:06 2024 -0400 - Log - Go SDK v2.58.0-RC1 --- No new revisions were added by this update. Summary of changes:
(beam) branch release-2.58.0 updated: Fix generateYamlDocs gradle task (#31909)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.58.0 by this push:
new 7626ad83d4e Fix generateYamlDocs gradle task (#31909)
7626ad83d4e is described below
commit 7626ad83d4e9182a5557ac52836c032b82f12e10
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Tue Jul 16 16:23:14 2024 -0400
Fix generateYamlDocs gradle task (#31909)
---
sdks/python/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index 72c696d30bf..832f567c66f 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -95,7 +95,7 @@ tasks.register("generateYamlDocs") {
dependsOn ":sdks:java:extensions:sql:expansion-service:shadowJar"
dependsOn ":sdks:java:io:expansion-service:build"
dependsOn ":sdks:java:io:google-cloud-platform:expansion-service:build"
- def extraPackages = "pyyaml markdown docstring_parser pandas pygments"
+ def extraPackages = "pyyaml markdown docstring_parser pandas pygments Jinja2"
doLast {
exec {
(beam) branch master updated: Fix generateYamlDocs gradle task (#31909)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 2d6d55b98ce Fix generateYamlDocs gradle task (#31909)
2d6d55b98ce is described below
commit 2d6d55b98cef5eacc4abd93863de2132cf89bea2
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Tue Jul 16 16:23:14 2024 -0400
Fix generateYamlDocs gradle task (#31909)
---
sdks/python/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index 72c696d30bf..832f567c66f 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -95,7 +95,7 @@ tasks.register("generateYamlDocs") {
dependsOn ":sdks:java:extensions:sql:expansion-service:shadowJar"
dependsOn ":sdks:java:io:expansion-service:build"
dependsOn ":sdks:java:io:google-cloud-platform:expansion-service:build"
- def extraPackages = "pyyaml markdown docstring_parser pandas pygments"
+ def extraPackages = "pyyaml markdown docstring_parser pandas pygments Jinja2"
doLast {
exec {
(beam) annotated tag sdks/v2.58.0-RC1 deleted (was f39c3f5f778)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag sdks/v2.58.0-RC1 was deleted! *** tag was f39c3f5f778 This change permanently discards the following revisions: discard b6aba7d9b97 Set version for 2.58.0 RC1
(beam) annotated tag v2.58.0-RC1 deleted (was 4a804951223)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag v2.58.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag v2.58.0-RC1 was deleted! *** tag was 4a804951223 This change permanently discards the following revisions: discard b6aba7d9b97 Set version for 2.58.0 RC1
svn commit: r70349 - in /dev/beam/2.58.0: ./ apache-beam-2.58.0-source-release.zip apache-beam-2.58.0-source-release.zip.asc apache-beam-2.58.0-source-release.zip.sha512
Author: jrmccluskey Date: Tue Jul 16 18:45:46 2024 New Revision: 70349 Log: Staging Java artifacts for Apache Beam 2.58.0 RC1 Added: dev/beam/2.58.0/ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip (with props) dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip == Binary file - no diff available. Propchange: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip -- svn:mime-type = application/octet-stream Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc == --- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc (added) +++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.asc Tue Jul 16 18:45:46 2024 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- + +iQJMBAABCgA2FiEEkTwzkqdwx4HtxN2r0gMW9xIhNCIFAmaWv9QYHHByaXZhdGVA +YmVhbS5hcGFjaGUub3JnAAoJENIDFvcSITQiQyIQAMHFAKjHh66owHM98d8tLZXv +x6JpzyILoWu36GahKOXBT1pCqyXguoKm01L/EpCBYFEcoICY8q4KaVgoQZvefRDe +/8c7pkaVXX25lbnfiKdzur2sXi6dsN8LP3P41yo9y7V6x+moWyMr/GcnsWPUBs6y +LDnp7DL2E+EhFpWyO4Iey+KG3KH8jttbGKTgGPcuh3aYFjsGpQRKsHsgYXIR3fAP +e4kK8dp3AZ26oHUIaLfH9HyxqLUB1HxqYViealy4AnIS6Q6AuoPhBi4oeFegyl+m +ZUNEX2eWXf9gA9G77SzeS3UF54NN+8SRbPabG7wvnRiyUGAYwcvug+bnhtYhrD02 +CRePK0ZLLCUyYVojVofvTLV/3fu5Qv+5jWudn48wWH2bNJn3fhbmFuf4qHm4OXgW +bWUmSbslQXuqmVFZbDXYYwplrG1cH55rR/ydwSoobvfiybUPu+O9ltPgQLTS1Hxh +xDW1zKnioXJm2UZik7f+62c5B+pfIWQ9mDPN4h+Jlj9PJDM1KWfCRx9NoWsvKlkc +g5Oo0xul+qF4+ER78h/+0bKhVNbkdo2nGsxPB9Di1LIyzkcLs5H8vHJB/un6IZjl +1gT4UDNdfr6ZHKb8MKMrAFyEx0kuXsDHXa405gZgRNNNQGbq2kUcFZP6thIERlgS +gvLNAdj8pQbw+bzRNdAR +=bT/s +-END PGP SIGNATURE- Added: dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 == --- dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 (added) +++ dev/beam/2.58.0/apache-beam-2.58.0-source-release.zip.sha512 Tue Jul 16 18:45:46 2024 @@ -0,0 +1 @@ +845a828fefeee560fb6b2d9bc318cc4a7c4682e1410053f7b5a03064becba40a6d585eb453b1a2af0a04563b95ae441d59c317ec255f86dc65ad7e191b9545af apache-beam-2.58.0-source-release.zip
(beam) annotated tag v2.58.0-RC1 updated (b6aba7d9b97 -> 4a804951223)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag v2.58.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag v2.58.0-RC1 was modified! *** from b6aba7d9b97 (commit) to 4a804951223 (tag) tagging b6aba7d9b977fb881eaeb2c3127eaf6984dad01d (commit) replaces jupyterlab-sidepanel-v3.0.0 by jrmccluskey on Tue Jul 16 18:35:34 2024 + - Log - v2.58.0-RC1 --- No new revisions were added by this update. Summary of changes:
(beam) annotated tag sdks/v2.58.0-RC1 updated (b6aba7d9b97 -> f39c3f5f778)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to annotated tag sdks/v2.58.0-RC1 in repository https://gitbox.apache.org/repos/asf/beam.git *** WARNING: tag sdks/v2.58.0-RC1 was modified! *** from b6aba7d9b97 (commit) to f39c3f5f778 (tag) tagging b6aba7d9b977fb881eaeb2c3127eaf6984dad01d (commit) replaces jupyterlab-sidepanel-v3.0.0 by jrmccluskey on Tue Jul 16 18:35:34 2024 + - Log - Go SDK v2.58.0-RC1 --- No new revisions were added by this update. Summary of changes:
(beam) branch release-2.58.0 updated: Export string sets in monitoring infos. (#31838) (#31849)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.58.0 by this push:
new 92de083fd98 Export string sets in monitoring infos. (#31838) (#31849)
92de083fd98 is described below
commit 92de083fd98445eb993a2c708a835b874a91f765
Author: Robert Bradshaw
AuthorDate: Tue Jul 16 10:45:01 2024 -0700
Export string sets in monitoring infos. (#31838) (#31849)
---
.../apache/beam/model/pipeline/v1/metrics.proto| 11 +
.../runners/core/metrics/MetricsContainerImpl.java | 53 +-
.../core/metrics/MonitoringInfoConstants.java | 2 +
.../core/metrics/SimpleMonitoringInfoBuilder.java | 11 +
.../core/metrics/MetricsContainerImplTest.java | 32 +
5 files changed, 108 insertions(+), 1 deletion(-)
diff --git
a/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/metrics.proto
b/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/metrics.proto
index 13c87bc1130..4ec189e4637 100644
---
a/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/metrics.proto
+++
b/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/metrics.proto
@@ -187,6 +187,17 @@ message MonitoringInfoSpecs {
}]
}];
+// Represents a set of strings seen across bundles.
+USER_SET_STRING = 21 [(monitoring_info_spec) = {
+ urn: "beam:metric:user:set_string:v1",
+ type: "beam:metrics:set_string:v1",
+ required_labels: ["PTRANSFORM", "NAMESPACE", "NAME"],
+ annotations: [{
+key: "description",
+value: "URN utilized to report user metric."
+ }]
+}];
+
// General monitored state information which contains structured
information
// which does not fit into a typical metric format. See MonitoringTableData
// for more details.
diff --git
a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
index a2f6511d512..99cf9850850 100644
---
a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
+++
b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/MetricsContainerImpl.java
@@ -19,13 +19,16 @@ package org.apache.beam.runners.core.metrics;
import static
org.apache.beam.runners.core.metrics.MonitoringInfoConstants.TypeUrns.DISTRIBUTION_INT64_TYPE;
import static
org.apache.beam.runners.core.metrics.MonitoringInfoConstants.TypeUrns.LATEST_INT64_TYPE;
+import static
org.apache.beam.runners.core.metrics.MonitoringInfoConstants.TypeUrns.SET_STRING_TYPE;
import static
org.apache.beam.runners.core.metrics.MonitoringInfoConstants.TypeUrns.SUM_INT64_TYPE;
import static
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.decodeInt64Counter;
import static
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.decodeInt64Distribution;
import static
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.decodeInt64Gauge;
+import static
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.decodeStringSet;
import static
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeInt64Counter;
import static
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeInt64Distribution;
import static
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeInt64Gauge;
+import static
org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.encodeStringSet;
import static
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
import java.io.Serializable;
@@ -331,6 +334,28 @@ public class MetricsContainerImpl implements Serializable,
MetricsContainer {
return builder.build();
}
+ /** @return The MonitoringInfo metadata from the string set metric. */
+ private @Nullable SimpleMonitoringInfoBuilder
stringSetToMonitoringMetadata(MetricKey metricKey) {
+return metricToMonitoringMetadata(
+metricKey,
+MonitoringInfoConstants.TypeUrns.SET_STRING_TYPE,
+MonitoringInfoConstants.Urns.USER_SET_STRING);
+ }
+
+ /**
+ * @param metricUpdate
+ * @return The MonitoringInfo generated from the string set metricUpdate.
+ */
+ private @Nullable MonitoringInfo stringSetUpdateToMonitoringInfo(
+ MetricUpdate metricUpdate) {
+SimpleMonitoringInfoBuilder builder =
stringSetToMonitoringMetadata(metricUpdate.getKey());
+if (builder == null) {
+ return null;
+}
+builder.setStringSetValue(metricUpdate.getUpdate());
+return builder.build();
+ }
+
/** Return the cumulative values for any metrics in this container as
MonitoringInfos. */
@Override
public Iterable getMoni
(beam) branch release-2.58.0 updated (fb96d6f42f1 -> dea262378cd)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch release-2.58.0 in repository https://gitbox.apache.org/repos/asf/beam.git from fb96d6f42f1 Cherrypicking #31721 to release 2.58.0 (#31868) add dea262378cd Cherrypicking #31837 (#31904) No new revisions were added by this update. Summary of changes: CHANGES.md | 3 ++- .../beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-)
(beam) branch release-2.58.0 updated: Cherrypicking #31721 to release 2.58.0 (#31868)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.58.0 by this push:
new fb96d6f42f1 Cherrypicking #31721 to release 2.58.0 (#31868)
fb96d6f42f1 is described below
commit fb96d6f42f1b6f49ac2d35f61e9cca524d497877
Author: Ahmed Abualsaud <65791736+ahmedab...@users.noreply.github.com>
AuthorDate: Mon Jul 15 10:48:52 2024 -0400
Cherrypicking #31721 to release 2.58.0 (#31868)
* add options to set min and max connections to connection management pool;
rename counter to be more accurate
* add multiplexing description
* add to CHANGES.md
* whitespace
* doc
* clarify documentation and address comments
* adjust description
* add details
---
CHANGES.md | 2 ++
.../beam/sdk/io/gcp/bigquery/AppendClientInfo.java | 8 +++
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 27 ++
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 9
4 files changed, 42 insertions(+), 4 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 96c436d89ec..fc94877a2bb 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -67,6 +67,7 @@
## New Features / Improvements
* Multiple RunInference instances can now share the same model instance by
setting the model_identifier parameter (Python)
([#31665](https://github.com/apache/beam/issues/31665)).
+* Added options to control the number of Storage API multiplexing connections
([#31721](https://github.com/apache/beam/pull/31721))
* [IcebergIO] All specified catalog properties are passed through to the
connector ([#31726](https://github.com/apache/beam/pull/31726))
* Removed a 3rd party LGPL dependency from the Go SDK
([#31765](https://github.com/apache/beam/issues/31765)).
* Support for MapState and SetState when using Dataflow Runner v1 with
Streaming Engine (Java)
([[#18200](https://github.com/apache/beam/issues/18200)])
@@ -82,6 +83,7 @@
## Bugfixes
+* Fixed a bug in BigQueryIO batch Storage Write API that frequently exhausted
concurrent connections quota
([#31710](https://github.com/apache/beam/pull/31710))
* Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
## Security Fixes
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java
index 5a12e81ea79..7505f77fb5b 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java
@@ -40,8 +40,8 @@ import org.apache.beam.sdk.metrics.Metrics;
*/
@AutoValue
abstract class AppendClientInfo {
- private final Counter activeConnections =
- Metrics.counter(AppendClientInfo.class, "activeConnections");
+ private final Counter activeStreamAppendClients =
+ Metrics.counter(AppendClientInfo.class, "activeStreamAppendClients");
abstract @Nullable BigQueryServices.StreamAppendClient
getStreamAppendClient();
@@ -123,7 +123,7 @@ abstract class AppendClientInfo {
writeStreamService.getStreamAppendClient(
streamName, getDescriptor(), useConnectionPool,
missingValueInterpretation);
- activeConnections.inc();
+ activeStreamAppendClients.inc();
return
toBuilder().setStreamName(streamName).setStreamAppendClient(client).build();
}
@@ -133,7 +133,7 @@ abstract class AppendClientInfo {
BigQueryServices.StreamAppendClient client = getStreamAppendClient();
if (client != null) {
getCloseAppendClient().accept(client);
- activeConnections.dec();
+ activeStreamAppendClients.dec();
}
}
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java
index cd1fc6d3842..ba76f483f77 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java
@@ -109,6 +109,28 @@ public interface BigQueryOptions
void setNumStorageWriteApiStreamAppendClients(Integer value);
+ @Description(
+ "When using the STORAGE_API_AT_LEAST_ONCE write method with multiplexing
(ie. useStorageApiConnectionPool=true), "
+ + "this option sets the minimum number of connections each pool
creates before any connections are shared. This is "
+ + "on a per worker, pe
(beam) branch release-2.58.0 updated (b95a750e862 -> d5a9bbdfe3c)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch release-2.58.0 in repository https://gitbox.apache.org/repos/asf/beam.git from b95a750e862 Merge pull request #31823 Add lineage information for BigQuery sinks. (#31836) add d5a9bbdfe3c Change orphan file log to warning (#31845) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/io/fileio.py | 14 +++--- 1 file changed, 7 insertions(+), 7 deletions(-)
(beam) branch release-2.58.0 updated: Merge pull request #31823 Add lineage information for BigQuery sinks. (#31836)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch release-2.58.0
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/release-2.58.0 by this push:
new b95a750e862 Merge pull request #31823 Add lineage information for
BigQuery sinks. (#31836)
b95a750e862 is described below
commit b95a750e862d5f9a179805528eff5dd0b605f597
Author: Robert Bradshaw
AuthorDate: Wed Jul 10 13:50:55 2024 -0700
Merge pull request #31823 Add lineage information for BigQuery sinks.
(#31836)
---
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 21 +++--
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java| 5 ++--
.../io/gcp/bigquery/BigQueryStorageSourceBase.java | 4 +--
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 8 -
.../bigquery/StorageApiWriteUnshardedRecords.java | 14 +
.../bigquery/StorageApiWritesShardedRecords.java | 6
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 6
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 6
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 34 +++---
9 files changed, 92 insertions(+), 12 deletions(-)
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
index 7f5d675ccf7..61bed66a336 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -54,6 +54,7 @@ import
org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.util.FluentBackoff;
import
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings;
import
org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
@@ -412,9 +413,23 @@ public class BigQueryHelpers {
return sb.toString();
}
- public static String dataCatalogName(TableReference ref) {
-return String.format(
-"bigquery:%s.%s.%s", ref.getProjectId(), ref.getDatasetId(),
ref.getTableId());
+ public static String dataCatalogName(TableReference ref, BigQueryOptions
options) {
+String tableIdBase;
+int ix = ref.getTableId().indexOf('$');
+if (ix == -1) {
+ tableIdBase = ref.getTableId();
+} else {
+ tableIdBase = ref.getTableId().substring(0, ix);
+}
+String projectId;
+if (!Strings.isNullOrEmpty(ref.getProjectId())) {
+ projectId = ref.getProjectId();
+} else if (!Strings.isNullOrEmpty(options.getBigQueryProject())) {
+ projectId = options.getBigQueryProject();
+} else {
+ projectId = options.getProject();
+}
+return String.format("bigquery:%s.%s.%s", projectId, ref.getDatasetId(),
tableIdBase);
}
static List getOrCreateMapListValue(Map> map, K key) {
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
index a863c49f46a..38c0c8e43b2 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
@@ -121,7 +121,7 @@ abstract class BigQuerySourceBase extends
BoundedSource {
BigQueryHelpers.toTableSpec(tableToExtract)));
}
// emit this table ID as a lineage source
-
Lineage.getSources().add(BigQueryHelpers.dataCatalogName(tableToExtract));
+ Lineage.getSources().add(BigQueryHelpers.dataCatalogName(tableToExtract,
bqOptions));
TableSchema schema = table.getSchema();
JobService jobService = bqServices.getJobService(bqOptions);
@@ -158,7 +158,8 @@ abstract class BigQuerySourceBase extends
BoundedSource {
if (res.extractedFiles.size() > 0) {
BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
// emit this table ID as a lineage source
-
Lineage.getSources().add(BigQueryHelpers.dataCatalogName(getTableToExtract(bqOptions)));
+Lineage.getSources()
+.add(BigQueryHelpers.dataCatalogName(getTableToExtract(bqOptions),
bqOptions));
final String extractDestinationDir =
resolveTempLocation(bqOptions.getTempLocation(),
"BigQueryExtractTemp", stepUuid);
/
(beam) branch master updated (b9a44126622 -> 6c829db657b)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from b9a44126622 Add changelog notes regarding Solace read connector (#31826) add 6c829db657b Bump google.golang.org/grpc from 1.64.0 to 1.64.1 in /sdks (#31817) No new revisions were added by this update. Summary of changes: sdks/go.mod | 2 +- sdks/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-)
(beam) branch master updated: Bump certifi from 2024.2.2 to 2024.7.4 in /sdks/python/container/py38 (#31790)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new a4558dfd8c8 Bump certifi from 2024.2.2 to 2024.7.4 in /sdks/python/container/py38 (#31790) a4558dfd8c8 is described below commit a4558dfd8c8ac46c32d5d121a66e24e7e36381b2 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Tue Jul 9 16:30:44 2024 -0400 Bump certifi from 2024.2.2 to 2024.7.4 in /sdks/python/container/py38 (#31790) Bumps [certifi](https://github.com/certifi/python-certifi) from 2024.2.2 to 2024.7.4. - [Commits](https://github.com/certifi/python-certifi/compare/2024.02.02...2024.07.04) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/python/container/py38/base_image_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt index 9cfd30ba6b3..f88dba10346 100644 --- a/sdks/python/container/py38/base_image_requirements.txt +++ b/sdks/python/container/py38/base_image_requirements.txt @@ -29,7 +29,7 @@ beautifulsoup4==4.12.3 bs4==0.0.2 build==1.2.1 cachetools==5.3.3 -certifi==2024.2.2 +certifi==2024.7.4 cffi==1.16.0 charset-normalizer==3.3.2 click==8.1.7
(beam) branch master updated: Add support for WindowStrategy Pane and AllowedLatness features (#31806)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 516bbc77ef3 Add support for WindowStrategy Pane and AllowedLatness
features (#31806)
516bbc77ef3 is described below
commit 516bbc77ef3886379983282ea53713bffba5290c
Author: Damon
AuthorDate: Tue Jul 9 08:50:49 2024 -0700
Add support for WindowStrategy Pane and AllowedLatness features (#31806)
---
sdks/go/pkg/beam/runners/prism/internal/handlerunner.go | 4
sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go | 4 +++-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
b/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
index a205c768731..910fa0a0ca8 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/handlerunner.go
@@ -297,6 +297,10 @@ func gbkBytes(ws *pipepb.WindowingStrategy, wc, kc, vc
*pipepb.Coder, toAggregat
outputTime = func(w typex.Window, et mtime.Time) mtime.Time {
return w.MaxTimestamp()
}
+ case pipepb.OutputTime_EARLIEST_IN_PANE,
pipepb.OutputTime_LATEST_IN_PANE:
+ outputTime = func(w typex.Window, et mtime.Time) mtime.Time {
+ return et
+ }
default:
// TODO need to correct session logic if output time is
different.
panic(fmt.Sprintf("unsupported OutputTime behavior: %v",
ws.GetOutputTime()))
diff --git a/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
b/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
index 737a1b22276..3efe48e2311 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
@@ -22,6 +22,7 @@ import (
"sync"
"sync/atomic"
+ "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime"
jobpb
"github.com/apache/beam/sdks/v2/go/pkg/beam/model/jobmanagement_v1"
pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1"
"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/urns"
@@ -195,7 +196,8 @@ func (s *Server) Prepare(ctx context.Context, req
*jobpb.PrepareJobRequest) (*jo
// Inspect Windowing strategies for unsupported features.
for wsID, ws := range
job.Pipeline.GetComponents().GetWindowingStrategies() {
- check("WindowingStrategy.AllowedLateness",
ws.GetAllowedLateness(), int64(0))
+ check("WindowingStrategy.AllowedLateness",
ws.GetAllowedLateness(), int64(0), mtime.MaxTimestamp.Milliseconds())
+
// Both Closing behaviors are identical without additional
trigger firings.
check("WindowingStrategy.ClosingBehaviour",
ws.GetClosingBehavior(), pipepb.ClosingBehavior_EMIT_IF_NONEMPTY,
pipepb.ClosingBehavior_EMIT_ALWAYS)
check("WindowingStrategy.AccumulationMode",
ws.GetAccumulationMode(), pipepb.AccumulationMode_DISCARDING)
(beam) branch master updated (8a88f1583f0 -> 746f3c5557e)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 8a88f1583f0 Solace Read connector: adding implementations of SempClient and SempClientFactory (#31542) add 746f3c5557e Use go 1.22 for self-hosted GHAs (#31767) No new revisions were added by this update. Summary of changes: .github/actions/setup-environment-action/action.yml | 6 -- .github/workflows/go_tests.yml | 7 +++ .github/workflows/local_env_tests.yml | 18 -- 3 files changed, 15 insertions(+), 16 deletions(-)
(beam) branch master updated: Bump cloud.google.com/go/storage from 1.41.0 to 1.43.0 in /sdks (#31772)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new d1df1d7ecc9 Bump cloud.google.com/go/storage from 1.41.0 to 1.43.0 in /sdks (#31772) d1df1d7ecc9 is described below commit d1df1d7ecc9f2b123586d07311f61cd3c34a2f77 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Mon Jul 8 09:59:26 2024 -0400 Bump cloud.google.com/go/storage from 1.41.0 to 1.43.0 in /sdks (#31772) Bumps [cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) from 1.41.0 to 1.43.0. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/spanner/v1.41.0...spanner/v1.43.0) --- updated-dependencies: - dependency-name: cloud.google.com/go/storage dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 16 sdks/go.sum | 34 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index b156c8beac2..b275e47f5c1 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -27,9 +27,9 @@ require ( cloud.google.com/go/bigtable v1.25.0 cloud.google.com/go/datastore v1.17.1 cloud.google.com/go/profiler v0.4.0 - cloud.google.com/go/pubsub v1.38.0 + cloud.google.com/go/pubsub v1.39.0 cloud.google.com/go/spanner v1.63.0 - cloud.google.com/go/storage v1.41.0 + cloud.google.com/go/storage v1.43.0 github.com/aws/aws-sdk-go-v2 v1.30.0 github.com/aws/aws-sdk-go-v2/config v1.27.4 github.com/aws/aws-sdk-go-v2/credentials v1.17.18 @@ -59,8 +59,8 @@ require ( golang.org/x/sync v0.7.0 golang.org/x/sys v0.21.0 golang.org/x/text v0.16.0 - google.golang.org/api v0.184.0 - google.golang.org/genproto v0.0.0-20240604185151-ef581f913117 + google.golang.org/api v0.187.0 + google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 @@ -74,7 +74,7 @@ require ( ) require ( - cloud.google.com/go/auth v0.5.1 // indirect + cloud.google.com/go/auth v0.6.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect dario.cat/mergo v1.0.0 // indirect filippo.io/edwards25519 v1.1.0 // indirect @@ -151,7 +151,7 @@ require ( github.com/google/renameio/v2 v2.0.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.4 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/gorilla/handlers v1.5.2 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -183,6 +183,6 @@ require ( golang.org/x/mod v0.17.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect ) diff --git a/sdks/go.sum b/sdks/go.sum index 2b2862bb2f0..4b8f7f0d458 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -99,8 +99,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= -cloud.google.com/go/auth v0.5.1 h1:0QNO7VThG54LUzKiQxv8C6x1YX7lUrzlAa1nVLF8CIw= -cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s= +cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38= +cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth
(beam) branch master updated: Fix installation invocation for OpenJDK (#31752)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new e28e00ac608 Fix installation invocation for OpenJDK (#31752) e28e00ac608 is described below commit e28e00ac60840535fd60596237223faf10c84c77 Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com> AuthorDate: Tue Jul 2 15:35:36 2024 -0400 Fix installation invocation for OpenJDK (#31752) --- playground/backend/containers/python/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/backend/containers/python/Dockerfile b/playground/backend/containers/python/Dockerfile index b617e02b15c..ca57bef9025 100644 --- a/playground/backend/containers/python/Dockerfile +++ b/playground/backend/containers/python/Dockerfile @@ -66,7 +66,7 @@ RUN cd /opt/playground/backend/kafka-emulator/ && tar -xvf kafka-emulator.tar && mv beam-playground-kafka-emulator-*.jar beam-playground-kafka-emulator.jar RUN apt-get update && \ wget http://http.us.debian.org/debian/pool/main/o/openjdk-11/openjdk-11-jre-headless_11.0.23+9-1~deb11u1_amd64.deb && \ -apt install -y ./openjdk-11-jre-headless_11.0.22+7-1~deb11u1_amd64.deb +apt install -y ./openjdk-11-jre-headless_11.0.23+9-1~deb11u1_amd64.deb # Create a user group `appgroup` and a user `appuser` RUN groupadd --gid 2 appgroup \
(beam) branch master updated: Update Playground Python Dockerfile (#31745)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new b5035a08836 Update Playground Python Dockerfile (#31745) b5035a08836 is described below commit b5035a08836c31451c5015c94f2a05fbb107febd Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com> AuthorDate: Tue Jul 2 15:12:19 2024 -0400 Update Playground Python Dockerfile (#31745) * Fix Python Playground Dockerfile * Update version to 11.0.23 --- playground/backend/containers/python/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/backend/containers/python/Dockerfile b/playground/backend/containers/python/Dockerfile index 709a22eec96..b617e02b15c 100644 --- a/playground/backend/containers/python/Dockerfile +++ b/playground/backend/containers/python/Dockerfile @@ -65,7 +65,7 @@ RUN cd /opt/playground/backend/kafka-emulator/ && tar -xvf kafka-emulator.tar && mv kafka-emulator/*.jar . && rmdir kafka-emulator/ &&\ mv beam-playground-kafka-emulator-*.jar beam-playground-kafka-emulator.jar RUN apt-get update && \ -wget http://http.us.debian.org/debian/pool/main/o/openjdk-11/openjdk-11-jre-headless_11.0.22+7-1~deb11u1_amd64.deb && \ +wget http://http.us.debian.org/debian/pool/main/o/openjdk-11/openjdk-11-jre-headless_11.0.23+9-1~deb11u1_amd64.deb && \ apt install -y ./openjdk-11-jre-headless_11.0.22+7-1~deb11u1_amd64.deb # Create a user group `appgroup` and a user `appuser`
(beam) branch master updated: Update Staticcheck to 2023.1.7 (#31750)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new dc63b0ffc99 Update Staticcheck to 2023.1.7 (#31750) dc63b0ffc99 is described below commit dc63b0ffc99b06e72bbfbb25c54f1cb8dbeb4968 Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com> AuthorDate: Tue Jul 2 13:50:08 2024 -0400 Update Staticcheck to 2023.1.7 (#31750) --- .github/workflows/go_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go_tests.yml b/.github/workflows/go_tests.yml index c7eed447095..db30bac68ec 100644 --- a/.github/workflows/go_tests.yml +++ b/.github/workflows/go_tests.yml @@ -65,6 +65,6 @@ jobs: go vet --copylocks=false --unsafeptr=false ./... - name: Run Staticcheck run: | - go install "honnef.co/go/tools/cmd/staticcheck@2023.1.3" + go install "honnef.co/go/tools/cmd/staticcheck@2023.1.7" cd sdks/go/pkg/beam $(go env GOPATH)/bin/staticcheck ./...
(beam) branch master updated (45733be4fe4 -> 8e11d0a4815)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 45733be4fe4 Do not build shadowjar and disable javadoc for it modules (#31741) add 8e11d0a4815 Update Go Version to 1.22.4 (#31747) No new revisions were added by this update. Summary of changes: .github/actions/setup-default-test-properties/test-properties.json| 2 +- .github/gh-actions-self-hosted-runners/arc/images/Dockerfile | 2 +- .github/workflows/go_tests.yml| 2 +- .github/workflows/local_env_tests.yml | 4 ++-- .../src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy| 2 +- dev-support/docker/Dockerfile | 2 +- sdks/go/run_with_go_version.sh| 2 +- 7 files changed, 8 insertions(+), 8 deletions(-)
(beam) branch master updated: Add a unit test for MakePipelineOptionsFileAndEnvVar (#31732)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new bae6fcfcb49 Add a unit test for MakePipelineOptionsFileAndEnvVar
(#31732)
bae6fcfcb49 is described below
commit bae6fcfcb492db64e8b7333d928fdbecdcb3e21f
Author: ljjulia
AuthorDate: Mon Jul 1 06:20:33 2024 -0700
Add a unit test for MakePipelineOptionsFileAndEnvVar (#31732)
* Add unit test for MakePipelineOptionsFileAndEnvVar
* Add coverage for validating options string is JSON
---
sdks/go/container/tools/pipeline_options.go| 5 +++
...ipeline_options.go => pipeline_options_test.go} | 49 +++---
2 files changed, 39 insertions(+), 15 deletions(-)
diff --git a/sdks/go/container/tools/pipeline_options.go
b/sdks/go/container/tools/pipeline_options.go
index 7b46d8fa8c8..026fb31b099 100644
--- a/sdks/go/container/tools/pipeline_options.go
+++ b/sdks/go/container/tools/pipeline_options.go
@@ -16,6 +16,7 @@
package tools
import (
+ "encoding/json"
"fmt"
"os"
)
@@ -31,6 +32,10 @@ func MakePipelineOptionsFileAndEnvVar(options string) error {
return fmt.Errorf("unable to create %v: %w", fn, err)
}
defer f.Close()
+ var js map[string]interface{}
+ if json.Unmarshal([]byte(options), ) != nil {
+ return fmt.Errorf("options string is not JSON formatted %v",
options)
+ }
if _, err := f.WriteString(options); err != nil {
return fmt.Errorf("error writing %v: %w", f.Name(), err)
}
diff --git a/sdks/go/container/tools/pipeline_options.go
b/sdks/go/container/tools/pipeline_options_test.go
similarity index 54%
copy from sdks/go/container/tools/pipeline_options.go
copy to sdks/go/container/tools/pipeline_options_test.go
index 7b46d8fa8c8..7a0d7ebd5f0 100644
--- a/sdks/go/container/tools/pipeline_options.go
+++ b/sdks/go/container/tools/pipeline_options_test.go
@@ -16,24 +16,43 @@
package tools
import (
- "fmt"
"os"
+ "testing"
)
-// MakePipelineOptionsFileAndEnvVar writes the pipeline options to a file.
-// Assumes the options string is JSON formatted.
-//
-// Stores the file name in question in PIPELINE_OPTIONS_FILE for access by the
SDK.
-func MakePipelineOptionsFileAndEnvVar(options string) error {
- fn := "pipeline_options.json"
- f, err := os.Create(fn)
- if err != nil {
- return fmt.Errorf("unable to create %v: %w", fn, err)
+func TestMakePipelineOptionsFileAndEnvVar(t *testing.T) {
+ tests := []struct {
+ name string
+ inputOptions string
+ expectedError string
+ }{
+ {
+ "empty options",
+ "{}",
+ "",
+ },
+ {
+ "valid options",
+ "{\"abc\": 123}",
+ "",
+ },
+ {
+ "invalid options",
+ "{4}",
+ "options string is not JSON formatted {4}",
+ },
}
- defer f.Close()
- if _, err := f.WriteString(options); err != nil {
- return fmt.Errorf("error writing %v: %w", f.Name(), err)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ t.Cleanup(os.Clearenv)
+ err :=
MakePipelineOptionsFileAndEnvVar(test.inputOptions)
+ if err != nil {
+ if got, want := err.Error(),
test.expectedError; got != want {
+ t.Errorf("got error: %v, want error:
%v", got, want)
+ }
+ }
+ })
}
- os.Setenv("PIPELINE_OPTIONS_FILE", f.Name())
- return nil
+ os.Remove("pipeline_options.json")
}
(beam-starter-go) 01/01: Merge pull request #38 from apache/dependabot/go_modules/github.com/apache/beam/sdks/v2-2.57.0
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/beam-starter-go.git commit 0f5af0e96cad628fe68b2b6de7763d01a5670c6f Merge: ccefb68 f375a73 Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com> AuthorDate: Thu Jun 27 11:19:14 2024 -0400 Merge pull request #38 from apache/dependabot/go_modules/github.com/apache/beam/sdks/v2-2.57.0 Bump github.com/apache/beam/sdks/v2 from 2.56.0 to 2.57.0 go.mod | 10 +- go.sum | 20 ++-- 2 files changed, 15 insertions(+), 15 deletions(-)
(beam-starter-go) branch main updated (ccefb68 -> 0f5af0e)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/beam-starter-go.git from ccefb68 Bump github.com/apache/beam/sdks/v2 from 2.55.1 to 2.56.0 (#37) add f375a73 Bump github.com/apache/beam/sdks/v2 from 2.56.0 to 2.57.0 new 0f5af0e Merge pull request #38 from apache/dependabot/go_modules/github.com/apache/beam/sdks/v2-2.57.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: go.mod | 10 +- go.sum | 20 ++-- 2 files changed, 15 insertions(+), 15 deletions(-)
(beam) branch master updated: Bump github.com/go-sql-driver/mysql from 1.8.0 to 1.8.1 in /sdks (#31688)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 4a5eaf21961 Bump github.com/go-sql-driver/mysql from 1.8.0 to 1.8.1 in /sdks (#31688) 4a5eaf21961 is described below commit 4a5eaf21961228188c8add70148a477b50af10a8 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Wed Jun 26 15:36:24 2024 -0400 Bump github.com/go-sql-driver/mysql from 1.8.0 to 1.8.1 in /sdks (#31688) Bumps [github.com/go-sql-driver/mysql](https://github.com/go-sql-driver/mysql) from 1.8.0 to 1.8.1. - [Release notes](https://github.com/go-sql-driver/mysql/releases) - [Changelog](https://github.com/go-sql-driver/mysql/blob/master/CHANGELOG.md) - [Commits](https://github.com/go-sql-driver/mysql/compare/v1.8.0...v1.8.1) --- updated-dependencies: - dependency-name: github.com/go-sql-driver/mysql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 2 +- sdks/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index 1109ee80086..e7443172de0 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -38,7 +38,7 @@ require ( github.com/aws/smithy-go v1.20.2 github.com/docker/go-connections v0.5.0 github.com/dustin/go-humanize v1.0.1 - github.com/go-sql-driver/mysql v1.8.0 + github.com/go-sql-driver/mysql v1.8.1 github.com/golang/protobuf v1.5.4 // TODO(danoliveira): Fully replace this with google.golang.org/protobuf github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 diff --git a/sdks/go.sum b/sdks/go.sum index 20234f53dd5..19fd97db5e7 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -827,8 +827,8 @@ github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.8.0 h1:UtktXaU2Nb64z/pLiGIxY4431SJ4/dR5cjMmlVHgnT4= -github.com/go-sql-driver/mysql v1.8.0/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
(beam) branch master updated (0f2e1963987 -> a944bf87cd0)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 0f2e1963987 Replace getElementType with getValueType for MAP in AvroGenericRecord… (#31653) add a944bf87cd0 Bump github.com/aws/aws-sdk-go-v2 from 1.27.2 to 1.30.0 in /sdks (#31670) No new revisions were added by this update. Summary of changes: sdks/go.mod | 2 +- sdks/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-)
(beam) branch master updated: Bump golang.org/x/net from 0.24.0 to 0.26.0 in /sdks (#31614)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 3695d496701 Bump golang.org/x/net from 0.24.0 to 0.26.0 in /sdks (#31614) 3695d496701 is described below commit 3695d496701cf65fd296ac2b046746aa48bb6526 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Fri Jun 21 10:07:54 2024 -0400 Bump golang.org/x/net from 0.24.0 to 0.26.0 in /sdks (#31614) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.24.0 to 0.26.0. - [Commits](https://github.com/golang/net/compare/v0.24.0...v0.26.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 14 +++--- sdks/go.sum | 28 ++-- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index 8ced6fbe1f9..ed1d980928c 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -54,11 +54,11 @@ require ( github.com/xitongsys/parquet-go v1.6.2 github.com/xitongsys/parquet-go-source v0.0.0-20220315005136-aec0fe3e777c go.mongodb.org/mongo-driver v1.13.1 - golang.org/x/net v0.24.0 + golang.org/x/net v0.26.0 golang.org/x/oauth2 v0.18.0 - golang.org/x/sync v0.6.0 - golang.org/x/sys v0.19.0 - golang.org/x/text v0.14.0 + golang.org/x/sync v0.7.0 + golang.org/x/sys v0.21.0 + golang.org/x/text v0.16.0 google.golang.org/api v0.171.0 google.golang.org/genproto v0.0.0-20240308144416-29370a3891b7 google.golang.org/grpc v1.63.2 @@ -181,9 +181,9 @@ require ( github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.22.0 // indirect - golang.org/x/mod v0.13.0 // indirect - golang.org/x/tools v0.14.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect diff --git a/sdks/go.sum b/sdks/go.sum index 20b520b5520..56ee80f8faa 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -494,8 +494,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -528,8 +528,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -553,8 +553,8 @@ golang.org/x/net v0.0.0-20201110031124-
(beam) branch master updated: Bump github.com/linkedin/goavro/v2 from 2.12.0 to 2.13.0 in /sdks (#31633)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 999b70b0c48 Bump github.com/linkedin/goavro/v2 from 2.12.0 to 2.13.0 in /sdks (#31633) 999b70b0c48 is described below commit 999b70b0c48dc76a73e9aa1006b4f0b9eda552d7 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Fri Jun 21 10:07:03 2024 -0400 Bump github.com/linkedin/goavro/v2 from 2.12.0 to 2.13.0 in /sdks (#31633) Bumps [github.com/linkedin/goavro/v2](https://github.com/linkedin/goavro) from 2.12.0 to 2.13.0. - [Release notes](https://github.com/linkedin/goavro/releases) - [Changelog](https://github.com/linkedin/goavro/blob/master/debug_release.go) - [Commits](https://github.com/linkedin/goavro/compare/v2.12...v2.13.0) --- updated-dependencies: - dependency-name: github.com/linkedin/goavro/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 2 +- sdks/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index ec96f146317..8ced6fbe1f9 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -44,7 +44,7 @@ require ( github.com/google/uuid v1.6.0 github.com/johannesboyne/gofakes3 v0.0.0-20221110173912-32fb85c5aed6 github.com/lib/pq v1.10.9 - github.com/linkedin/goavro/v2 v2.12.0 + github.com/linkedin/goavro/v2 v2.13.0 github.com/nats-io/nats-server/v2 v2.10.12 github.com/nats-io/nats.go v1.33.1 github.com/proullon/ramsql v0.1.3 diff --git a/sdks/go.sum b/sdks/go.sum index bb0d3286d8e..20b520b5520 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -327,8 +327,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linkedin/goavro/v2 v2.12.0 h1:rIQQSj8jdAUlKQh6DttK8wCRv4t4QO09g1C4aBWXslg= -github.com/linkedin/goavro/v2 v2.12.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk= +github.com/linkedin/goavro/v2 v2.13.0 h1:L8eI8GcuciwUkt41Ej62joSZS4kKaYIUdze+6for9NU= +github.com/linkedin/goavro/v2 v2.13.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
(beam) branch master updated: Update documentation for new TFT Transforms (#31565)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9c80b81fd47 Update documentation for new TFT Transforms (#31565)
9c80b81fd47 is described below
commit 9c80b81fd47a2b8f13db65cbc9dde8cf007af5e8
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Wed Jun 12 12:04:20 2024 -0400
Update documentation for new TFT Transforms (#31565)
* Update documentation for new TFT Transforms
* narrow versioning scope
---
sdks/python/apache_beam/ml/transforms/tft.py| 4
website/www/site/content/en/documentation/ml/preprocess-data.md | 7 ++-
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/sdks/python/apache_beam/ml/transforms/tft.py
b/sdks/python/apache_beam/ml/transforms/tft.py
index e03d52214c9..6903cca8941 100644
--- a/sdks/python/apache_beam/ml/transforms/tft.py
+++ b/sdks/python/apache_beam/ml/transforms/tft.py
@@ -52,13 +52,17 @@ __all__ = [
'ComputeAndApplyVocabulary',
'ScaleToZScore',
'ScaleTo01',
+'ScaleToGaussian',
'ApplyBuckets',
+'ApplyBucketsWithInterpolation',
'Bucketize',
'TFIDF',
'TFTOperation',
'ScaleByMinMax',
'NGrams',
'BagOfWords',
+'HashStrings',
+'DeduplicateTensorPerRow',
]
# Register the expected input types for each operation
diff --git a/website/www/site/content/en/documentation/ml/preprocess-data.md
b/website/www/site/content/en/documentation/ml/preprocess-data.md
index 1365926d3cc..fb21cc8928b 100644
--- a/website/www/site/content/en/documentation/ml/preprocess-data.md
+++ b/website/www/site/content/en/documentation/ml/preprocess-data.md
@@ -43,7 +43,7 @@ available transforms, see the [Transforms](#transforms)
section on this page.
## Support and limitations {#support}
- Available in the Apache Beam Python SDK versions 2.53.0 and later.
-- Supports Python 3.8, 3.9, and 3.10.
+- Supports Python 3.8, 3.9, 3.10, and 3.11
- Only available for pipelines that use [default
windows](/documentation/programming-guide/#single-global-window).
## Transforms {#transforms}
@@ -74,11 +74,16 @@ TensorFlow documentation.
| Transform name | Description |
| --- | ---|
| ApplyBuckets | See
[`tft.apply_buckets`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/apply_buckets)
in the TensorFlow documentation. |
+| ApplyBucketsWithInterpolation | See
[`tft.apply_buckets_with_interpolation`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/apply_buckets_with_interpolation)
in the TensorFlow documentation. |
+| BagOfWords | See
[`tft.bag_of_words`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/bag_of_words)
in the TensorFlow documentation. |
| Bucketize | See
[`tft.bucketize`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/bucketize)
in the TensorFlow documentation. |
| ComputeAndApplyVocabulary | See
[`tft.compute_and_apply_vocabulary`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/compute_and_apply_vocabulary)
in the TensorFlow documentation. |
+| DeduplicateTensorPerRow | See
[`tft.deduplicate_tensor_per_row`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/deduplicate_tensor_per_row)
in the TensorFlow documentation. |
+| HashStrings | See
[`tft.hash_strings`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/hash_strings)
in the TensorFlow documentation. |
| NGrams | See
[`tft.ngrams`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/ngrams)
in the TensorFlow documentation. |
| ScaleByMinMax | See
[`tft.scale_by_min_max`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_by_min_max)
in the TensorFlow documentation. |
| ScaleTo01 | See
[`tft.scale_to_0_1`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_to_0_1)
in the TensorFlow documentation. |
+| ScaleToGaussian | See
[`tft.scale_to_gaussian`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_to_gaussian)
in the TensorFlow documentation. |
| ScaleToZScore | See
[`tft.scale_to_z_score`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_to_z_score)
in the TensorFlow documentation. |
| TFIDF | See
[`tft.tfidf`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/tfidf)
in the TensorFlow documentation. |:
{{< /table >}}
(beam) branch master updated: Add image embedding support to TFHub MLTransforms (#31564)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 7732e11acb7 Add image embedding support to TFHub MLTransforms (#31564) 7732e11acb7 is described below commit 7732e11acb7e215649748f0cff32212a4c0d777d Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com> AuthorDate: Wed Jun 12 12:03:45 2024 -0400 Add image embedding support to TFHub MLTransforms (#31564) * Add image embedding support to TFHub MLTransforms * linting * more linting * formatting * typo --- .../ml/transforms/embeddings/tensorflow_hub.py | 42 ++- .../transforms/embeddings/tensorflow_hub_test.py | 60 ++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py b/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py index 9e448078825..f78ddf3ff04 100644 --- a/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py +++ b/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub.py @@ -29,9 +29,10 @@ from apache_beam.ml.inference.base import RunInference from apache_beam.ml.inference.tensorflow_inference import TFModelHandlerTensor from apache_beam.ml.inference.tensorflow_inference import default_tensor_inference_fn from apache_beam.ml.transforms.base import EmbeddingsManager +from apache_beam.ml.transforms.base import _ImageEmbeddingHandler from apache_beam.ml.transforms.base import _TextEmbeddingHandler -__all__ = ['TensorflowHubTextEmbeddings'] +__all__ = ['TensorflowHubTextEmbeddings', 'TensorflowHubImageEmbeddings'] # TODO: https://github.com/apache/beam/issues/30288 @@ -132,3 +133,42 @@ class TensorflowHubTextEmbeddings(EmbeddingsManager): model_handler=_TextEmbeddingHandler(self), inference_args=self.inference_args, )) + + +class TensorflowHubImageEmbeddings(EmbeddingsManager): + def __init__(self, columns: List[str], hub_url: str, **kwargs): +""" +Embedding config for tensorflow hub models. This config can be used with +MLTransform to embed image data. Models are loaded using the RunInference +PTransform with the help of a ModelHandler. + +Args: + columns: The columns containing the images to be embedded. + hub_url: The url of the tensorflow hub model. + min_batch_size: The minimum batch size to be used for inference. + max_batch_size: The maximum batch size to be used for inference. + large_model: Whether to share the model across processes. +""" +super().__init__(columns=columns, **kwargs) +self.model_uri = hub_url + + def get_model_handler(self) -> ModelHandler: +# override the default inference function +return _TensorflowHubModelHandler( +model_uri=self.model_uri, +preprocessing_url=None, +min_batch_size=self.min_batch_size, +max_batch_size=self.max_batch_size, +large_model=self.large_model, +) + + def get_ptransform_for_processing(self, **kwargs) -> beam.PTransform: +""" +Returns a RunInference object that is used to run inference on the text +input using _ImageEmbeddingHandler. +""" +return ( +RunInference( +model_handler=_ImageEmbeddingHandler(self), +inference_args=self.inference_args, +)) diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py b/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py index b08ca8e2d8e..24bca5155fa 100644 --- a/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py +++ b/sdks/python/apache_beam/ml/transforms/embeddings/tensorflow_hub_test.py @@ -20,10 +20,13 @@ import tempfile import unittest import uuid +import numpy as np + import apache_beam as beam from apache_beam.ml.transforms.base import MLTransform hub_url = 'https://tfhub.dev/google/nnlm-en-dim128/2' +hub_img_url = 'https://www.kaggle.com/models/google/resnet-v2/TensorFlow2/101-feature-vector/2' # pylint: disable=line-too-long test_query_column = 'test_query' test_query = 'This is a test query' @@ -32,6 +35,7 @@ try: from apache_beam.ml.transforms.embeddings.tensorflow_hub import TensorflowHubTextEmbeddings except ImportError: TensorflowHubTextEmbeddings = None # type: ignore + tf = None # pylint: disable=ungrouped-imports try: @@ -40,6 +44,14 @@ try: except ImportError: tft = None +# pylint: disable=ungrouped-imports +try: + from apache_beam.ml.transforms.embeddings.tensorflow_hub import TensorflowHubImageEmbeddings + from PIL import Image +except ImportError: + TensorflowHubImageEmbeddings = None # type: ignore + Image = None +
(beam) branch master updated: Update Python version in the dev environment script. (#31416)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 6a24b07b6c6 Update Python version in the dev environment script. (#31416) 6a24b07b6c6 is described below commit 6a24b07b6c69f4079202f8740a1e42da8494506c Author: tvalentyn AuthorDate: Tue Jun 11 08:24:15 2024 -0700 Update Python version in the dev environment script. (#31416) * Update Python version in the dev environment script. * fixup --- local-env-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/local-env-setup.sh b/local-env-setup.sh index 6cd1092023a..f13dc88432a 100755 --- a/local-env-setup.sh +++ b/local-env-setup.sh @@ -55,7 +55,7 @@ if [ "$kernelname" = "Linux" ]; then exit fi -for ver in 3.8 3.9 3.10 3.11 3; do +for ver in 3.8 3.9 3.10 3.11 3.12 3; do apt install --yes python$ver-venv done @@ -89,7 +89,7 @@ elif [ "$kernelname" = "Darwin" ]; then echo "Installing openjdk@8" brew install openjdk@8 fi -for ver in 3.8 3.9 3.10 3.11; do +for ver in 3.8 3.9 3.10 3.11 3.12; do if brew ls --versions python@$ver > /dev/null; then echo "python@$ver already installed. Skipping" brew info python@$ver
(beam) branch master updated: Implement Hugging Face Image Embedding MLTransform (#31536)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new fe54c211512 Implement Hugging Face Image Embedding MLTransform (#31536)
fe54c211512 is described below
commit fe54c211512ae90f0d596bf2ed1e40d3826a7c7e
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Fri Jun 7 11:21:02 2024 -0400
Implement Hugging Face Image Embedding MLTransform (#31536)
* Implement Hugging Face Image Embedding MLTransform
* correct imports
* Simplify to original sentence transformer class
---
.../ml/transforms/embeddings/huggingface.py| 19 ++--
.../ml/transforms/embeddings/huggingface_test.py | 52 +-
2 files changed, 67 insertions(+), 4 deletions(-)
diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/huggingface.py
b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface.py
index 7fcaa9c9a5d..46b4ef9cf7d 100644
--- a/sdks/python/apache_beam/ml/transforms/embeddings/huggingface.py
+++ b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface.py
@@ -32,6 +32,7 @@ import apache_beam as beam
from apache_beam.ml.inference.base import ModelHandler
from apache_beam.ml.inference.base import RunInference
from apache_beam.ml.transforms.base import EmbeddingsManager
+from apache_beam.ml.transforms.base import _ImageEmbeddingHandler
from apache_beam.ml.transforms.base import _TextEmbeddingHandler
try:
@@ -114,6 +115,7 @@ class SentenceTransformerEmbeddings(EmbeddingsManager):
model_name: str,
columns: List[str],
max_seq_length: Optional[int] = None,
+ image_model: bool = False,
**kwargs):
"""
Embedding config for sentence-transformers. This config can be used with
@@ -122,9 +124,13 @@ class SentenceTransformerEmbeddings(EmbeddingsManager):
Args:
model_name: Name of the model to use. The model should be hosted on
-HuggingFace Hub or compatible with sentence_transformers.
+HuggingFace Hub or compatible with sentence_transformers. For image
+embedding models, see
+
https://www.sbert.net/docs/sentence_transformer/pretrained_models.html#image-text-models
# pylint: disable=line-too-long
+for a list of available sentence_transformers models.
columns: List of columns to be embedded.
max_seq_length: Max sequence length to use for the model if applicable.
+ image_model: Whether the model is generating image embeddings.
min_batch_size: The minimum batch size to be used for inference.
max_batch_size: The maximum batch size to be used for inference.
large_model: Whether to share the model across processes.
@@ -132,6 +138,7 @@ class SentenceTransformerEmbeddings(EmbeddingsManager):
super().__init__(columns, **kwargs)
self.model_name = model_name
self.max_seq_length = max_seq_length
+self.image_model = image_model
def get_model_handler(self):
return _SentenceTransformerModelHandler(
@@ -144,8 +151,14 @@ class SentenceTransformerEmbeddings(EmbeddingsManager):
large_model=self.large_model)
def get_ptransform_for_processing(self, **kwargs) -> beam.PTransform:
-# wrap the model handler in a _TextEmbeddingHandler since
-# the SentenceTransformerEmbeddings works on text input data.
+# wrap the model handler in an appropriate embedding handler to provide
+# some type checking.
+if self.image_model:
+ return (
+ RunInference(
+ model_handler=_ImageEmbeddingHandler(self),
+ inference_args=self.inference_args,
+ ))
return (
RunInference(
model_handler=_TextEmbeddingHandler(self),
diff --git
a/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py
b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py
index f94e747c5ed..d09a573b676 100644
--- a/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py
+++ b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py
@@ -35,6 +35,7 @@ from apache_beam.testing.util import equal_to
try:
from apache_beam.ml.transforms.embeddings.huggingface import
SentenceTransformerEmbeddings
from apache_beam.ml.transforms.embeddings.huggingface import
InferenceAPIEmbeddings
+ from PIL import Image
import torch
except ImportError:
SentenceTransformerEmbeddings = None # type: ignore
@@ -46,10 +47,17 @@ try:
except ImportError:
tft = None
+# pylint: disable=ungrouped-imports
+try:
+ from PIL import Image
+except ImportError:
+ Image = None
+
_HF_TOKEN = os.environ.get('HF_INFERENCE_TOKEN')
test_query = "This is a test"
test_query_column = "feature_1"
DEFAULT_MODEL_NAME = "sentence-transformers/all-
(beam) branch master updated: Bump tqdm from 4.66.2 to 4.66.3 in /sdks/python/container/py38 (#31179)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 9edd413ef73 Bump tqdm from 4.66.2 to 4.66.3 in /sdks/python/container/py38 (#31179) 9edd413ef73 is described below commit 9edd413ef734208e04e8be2b4e71be713b8a56a8 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Fri Jun 7 11:16:50 2024 -0400 Bump tqdm from 4.66.2 to 4.66.3 in /sdks/python/container/py38 (#31179) Bumps [tqdm](https://github.com/tqdm/tqdm) from 4.66.2 to 4.66.3. - [Release notes](https://github.com/tqdm/tqdm/releases) - [Commits](https://github.com/tqdm/tqdm/compare/v4.66.2...v4.66.3) --- updated-dependencies: - dependency-name: tqdm dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/python/container/py38/base_image_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt index c87b4fac4b2..9cfd30ba6b3 100644 --- a/sdks/python/container/py38/base_image_requirements.txt +++ b/sdks/python/container/py38/base_image_requirements.txt @@ -151,7 +151,7 @@ tenacity==8.2.3 testcontainers==3.7.1 threadpoolctl==3.4.0 tomli==2.0.1 -tqdm==4.66.2 +tqdm==4.66.3 typing_extensions==4.11.0 tzdata==2024.1 tzlocal==5.2
(beam) branch master updated (10b6342c260 -> 47d9bca8bf1)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 10b6342c260 Fix assert message, fix streaming mode (#31540) add 47d9bca8bf1 Fix Python Linting Errors on Master (#31546) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/internal/gcp/auth.py | 4 ++-- sdks/python/apache_beam/internal/gcp/auth_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-)
(beam) branch master updated: Fix assert message, fix streaming mode (#31540)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 10b6342c260 Fix assert message, fix streaming mode (#31540)
10b6342c260 is described below
commit 10b6342c26005d52963d410efc1a78e4bd03e888
Author: Vitaly Terentyev
AuthorDate: Fri Jun 7 17:32:31 2024 +0400
Fix assert message, fix streaming mode (#31540)
---
.../org/apache/beam/it/gcp/bigquery/BigQueryIOST.java | 19 ---
1 file changed, 16 insertions(+), 3 deletions(-)
diff --git
a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java
b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java
index a49df8a..22ff94e293b 100644
---
a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java
+++
b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryIOST.java
@@ -296,6 +296,11 @@ public final class BigQueryIOST extends IOStressTestBase {
.withSchema(schema)
.withCustomGcsTempLocation(ValueProvider.StaticValueProvider.of(tempLocation)));
+String experiments =
+configuration.writeMethod.equals(STORAGE_API_AT_LEAST_ONCE_METHOD)
+? GcpOptions.STREAMING_ENGINE_EXPERIMENT +
",streaming_mode_at_least_once"
+: GcpOptions.STREAMING_ENGINE_EXPERIMENT;
+
PipelineLauncher.LaunchConfig options =
PipelineLauncher.LaunchConfig.builder("write-bigquery")
.setSdk(PipelineLauncher.Sdk.JAVA)
@@ -307,7 +312,7 @@ public final class BigQueryIOST extends IOStressTestBase {
.toString())
.addParameter("numWorkers",
String.valueOf(configuration.numWorkers))
.addParameter("maxNumWorkers",
String.valueOf(configuration.maxNumWorkers))
-.addParameter("experiments",
GcpOptions.STREAMING_ENGINE_EXPERIMENT)
+.addParameter("experiments", experiments)
.build();
PipelineLauncher.LaunchInfo launchInfo = pipelineLauncher.launch(project,
region, options);
@@ -329,9 +334,17 @@ public final class BigQueryIOST extends IOStressTestBase {
// Depending on writing method there might be duplicates on different
sides (read or write).
if (configuration.writeMethod.equals(STORAGE_API_AT_LEAST_ONCE_METHOD)) {
- assertTrue(rowCount >= numRecords);
+ assertTrue(
+ String.format(
+ "Number of rows in the table (%d) is less than the expected
number (%d). Missing records: %d",
+ rowCount, (long) numRecords, (long) numRecords - rowCount),
+ rowCount >= numRecords);
} else {
- assertTrue(numRecords >= rowCount);
+ assertTrue(
+ String.format(
+ "Number of rows in the table (%d) is greater than the expected
number (%d).",
+ rowCount, (long) numRecords),
+ numRecords >= rowCount);
}
// export metrics
(beam) branch master updated: Deduplicate Base Embedding Handler Code (#31534)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9a921d51054 Deduplicate Base Embedding Handler Code (#31534)
9a921d51054 is described below
commit 9a921d51054b7cf1be8119c76dabca80d26c9bfc
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Fri Jun 7 09:26:36 2024 -0400
Deduplicate Base Embedding Handler Code (#31534)
* Deduplicate Base Embedding Handler Code
* linting
---
sdks/python/apache_beam/ml/transforms/base.py | 131 +-
1 file changed, 43 insertions(+), 88 deletions(-)
diff --git a/sdks/python/apache_beam/ml/transforms/base.py
b/sdks/python/apache_beam/ml/transforms/base.py
index 0d7db282bb8..678ab0882d2 100644
--- a/sdks/python/apache_beam/ml/transforms/base.py
+++ b/sdks/python/apache_beam/ml/transforms/base.py
@@ -587,9 +587,9 @@ class _MLTransformToPTransformMapper:
return _transform_attribute_manager.load_attributes(artifact_location)
-class _TextEmbeddingHandler(ModelHandler):
+class _EmbeddingHandler(ModelHandler):
"""
- A ModelHandler intended to be work on list[dict[str, str]] inputs.
+ A ModelHandler intended to be work on list[dict[str, Any]] inputs.
The inputs to the model handler are expected to be a list of dicts.
@@ -597,12 +597,10 @@ class _TextEmbeddingHandler(ModelHandler):
PCollection[E] to a PCollection[P], this ModelHandler would take a
PCollection[Dict[str, E]] to a PCollection[Dict[str, P]].
- _TextEmbeddingHandler will accept an EmbeddingsManager instance, which
+ _EmbeddingHandler will accept an EmbeddingsManager instance, which
contains the details of the model to be loaded and the inference_fn to be
- used. The purpose of _TextEmbeddingHandler is to generate embeddings for
- text inputs using the EmbeddingsManager instance.
-
- If the input is not a text column, a RuntimeError will be raised.
+ used. The purpose of _EmbeddingHandler is to generate embeddings for
+ general inputs using the EmbeddingsManager instance.
This is an internal class and offers no backwards compatibility guarantees.
@@ -619,12 +617,9 @@ class _TextEmbeddingHandler(ModelHandler):
return model
def _validate_column_data(self, batch):
-if not isinstance(batch[0], (str, bytes)):
- raise TypeError(
- 'Embeddings can only be generated on Dict[str, str].'
- f'Got Dict[str, {type(batch[0])}] instead.')
+pass
- def _validate_batch(self, batch: Sequence[Dict[str, List[str]]]):
+ def _validate_batch(self, batch: Sequence[Dict[str, Any]]):
if not batch or not isinstance(batch[0], dict):
raise TypeError(
'Expected data to be dicts, got '
@@ -676,8 +671,7 @@ class _TextEmbeddingHandler(ModelHandler):
def get_metrics_namespace(self) -> str:
return (
-self._underlying.get_metrics_namespace() or
-'BeamML_TextEmbeddingHandler')
+self._underlying.get_metrics_namespace() or 'BeamML_EmbeddingHandler')
def batch_elements_kwargs(self) -> Mapping[str, Any]:
batch_sizes_map = {}
@@ -694,7 +688,41 @@ class _TextEmbeddingHandler(ModelHandler):
pass
-class _ImageEmbeddingHandler(ModelHandler):
+class _TextEmbeddingHandler(_EmbeddingHandler):
+ """
+ A ModelHandler intended to be work on list[dict[str, str]] inputs.
+
+ The inputs to the model handler are expected to be a list of dicts.
+
+ For example, if the original mode is used with RunInference to take a
+ PCollection[E] to a PCollection[P], this ModelHandler would take a
+ PCollection[Dict[str, E]] to a PCollection[Dict[str, P]].
+
+ _TextEmbeddingHandler will accept an EmbeddingsManager instance, which
+ contains the details of the model to be loaded and the inference_fn to be
+ used. The purpose of _TextEmbeddingHandler is to generate embeddings for
+ text inputs using the EmbeddingsManager instance.
+
+ If the input is not a text column, a RuntimeError will be raised.
+
+ This is an internal class and offers no backwards compatibility guarantees.
+
+ Args:
+embeddings_manager: An EmbeddingsManager instance.
+ """
+ def _validate_column_data(self, batch):
+if not isinstance(batch[0], (str, bytes)):
+ raise TypeError(
+ 'Embeddings can only be generated on Dict[str, str].'
+ f'Got Dict[str, {type(batch[0])}] instead.')
+
+ def get_metrics_namespace(self) -> str:
+return (
+self._underlying.get_metrics_namespace() or
+'BeamML_TextEmbeddingHandler')
+
+
+class _ImageEmbeddingHandler(_EmbeddingHandler):
"""
A ModelHandler intended to be work on list[dict[str, Image]] inputs.
@@ -717,15 +745,6 @@ class _ImageEmbeddingHandler(ModelHandler):
Args:
embeddings_manager: An
(beam) branch master updated (8c6e1a4654b -> 3cdbd3a908a)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 8c6e1a4654b Add missing import (#31533) add 3cdbd3a908a Implement Vertex AI Image Embedding MLTransform (#31514) No new revisions were added by this update. Summary of changes: .../ml/transforms/embeddings/vertex_ai.py | 96 +- .../ml/transforms/embeddings/vertex_ai_test.py | 41 + 2 files changed, 136 insertions(+), 1 deletion(-)
(beam) branch master updated: update the transform label error message (#31508)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new bd00a69402f update the transform label error message (#31508) bd00a69402f is described below commit bd00a69402f45520c193afd82bea51b31b90f681 Author: liferoad AuthorDate: Wed Jun 5 10:45:34 2024 -0400 update the transform label error message (#31508) --- sdks/python/apache_beam/pipeline.py | 6 -- sdks/python/apache_beam/pipeline_test.py | 6 -- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py index 11bc74d27ec..a20fae2ae73 100644 --- a/sdks/python/apache_beam/pipeline.py +++ b/sdks/python/apache_beam/pipeline.py @@ -699,8 +699,10 @@ class Pipeline(HasDisplayData): else: raise RuntimeError( 'A transform with label "%s" already exists in the pipeline. ' -'To apply a transform with a specified label write ' -'pvalue | "label" >> transform' % full_label) +'To apply a transform with a specified label, write ' +'pvalue | "label" >> transform or use the option ' +'"auto_unique_labels" to automatically generate unique ' +'transform labels' % full_label) self.applied_labels.add(full_label) pvalueish, inputs = transform._extract_input_pvalues(pvalueish) diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py index 38d8b1f03a1..a11f2e09ffc 100644 --- a/sdks/python/apache_beam/pipeline_test.py +++ b/sdks/python/apache_beam/pipeline_test.py @@ -264,8 +264,10 @@ class PipelineTest(unittest.TestCase): self.assertEqual( cm.exception.args[0], 'A transform with label "CustomTransform" already exists in the ' -'pipeline. To apply a transform with a specified label write ' -'pvalue | "label" >> transform') +'pipeline. To apply a transform with a specified label, write ' +'pvalue | "label" >> transform or use the option ' +'"auto_unique_labels" to automatically generate unique ' +'transform labels') def test_auto_unique_labels(self):
(beam) branch master updated: Add base _ImageEmbeddingHandler (#31501)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 7f51d6a9bdf Add base _ImageEmbeddingHandler (#31501)
7f51d6a9bdf is described below
commit 7f51d6a9bdf433158a8a6564a0154d8f16571847
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Wed Jun 5 10:43:46 2024 -0400
Add base _ImageEmbeddingHandler (#31501)
* Add base _ImageEmbeddingHandler
* add pillow dependency for testing
* Add skips if dependency not installed
* remove extra base dependencies
* fix import check
* further clean up image import
* linting
* typo
---
sdks/python/apache_beam/ml/transforms/base.py | 110 +
sdks/python/apache_beam/ml/transforms/base_test.py | 88 +
sdks/python/setup.py | 1 +
3 files changed, 199 insertions(+)
diff --git a/sdks/python/apache_beam/ml/transforms/base.py
b/sdks/python/apache_beam/ml/transforms/base.py
index a297f62f5bc..0d7db282bb8 100644
--- a/sdks/python/apache_beam/ml/transforms/base.py
+++ b/sdks/python/apache_beam/ml/transforms/base.py
@@ -692,3 +692,113 @@ class _TextEmbeddingHandler(ModelHandler):
def validate_inference_args(self, _):
pass
+
+
+class _ImageEmbeddingHandler(ModelHandler):
+ """
+ A ModelHandler intended to be work on list[dict[str, Image]] inputs.
+
+ The inputs to the model handler are expected to be a list of dicts.
+
+ For example, if the original mode is used with RunInference to take a
+ PCollection[E] to a PCollection[P], this ModelHandler would take a
+ PCollection[Dict[str, E]] to a PCollection[Dict[str, P]].
+
+ _ImageEmbeddingHandler will accept an EmbeddingsManager instance, which
+ contains the details of the model to be loaded and the inference_fn to be
+ used. The purpose of _ImageEmbeddingHandler is to generate embeddings for
+ image inputs using the EmbeddingsManager instance.
+
+ If the input is not an Image representation column, a RuntimeError will be
+ raised.
+
+ This is an internal class and offers no backwards compatibility guarantees.
+
+ Args:
+embeddings_manager: An EmbeddingsManager instance.
+ """
+ def __init__(self, embeddings_manager: EmbeddingsManager):
+self.embedding_config = embeddings_manager
+self._underlying = self.embedding_config.get_model_handler()
+self.columns = self.embedding_config.get_columns_to_apply()
+
+ def load_model(self):
+model = self._underlying.load_model()
+return model
+
+ def _validate_column_data(self, batch):
+# Don't want to require framework-specific imports
+# here, so just catch columns of primatives for now.
+if isinstance(batch[0], (int, str, float, bool)):
+ raise TypeError(
+ 'Embeddings can only be generated on Dict[str, Image].'
+ f'Got Dict[str, {type(batch[0])}] instead.')
+
+ def _validate_batch(self, batch: Sequence[Dict[str, List[Any]]]):
+if not batch or not isinstance(batch[0], dict):
+ raise TypeError(
+ 'Expected data to be dicts, got '
+ f'{type(batch[0])} instead.')
+
+ def _process_batch(
+ self,
+ dict_batch: Dict[str, List[Any]],
+ model: ModelT,
+ inference_args: Optional[Dict[str, Any]]) -> Dict[str, List[Any]]:
+result: Dict[str, List[Any]] = collections.defaultdict(list)
+input_keys = dict_batch.keys()
+missing_columns_in_data = set(self.columns) - set(input_keys)
+if missing_columns_in_data:
+ raise RuntimeError(
+ f'Data does not contain the following columns '
+ f': {missing_columns_in_data}.')
+for key, batch in dict_batch.items():
+ if key in self.columns:
+self._validate_column_data(batch)
+prediction = self._underlying.run_inference(
+batch, model, inference_args)
+if isinstance(prediction, np.ndarray):
+ prediction = prediction.tolist()
+ result[key] = prediction # type: ignore[assignment]
+else:
+ result[key] = prediction # type: ignore[assignment]
+ else:
+result[key] = batch
+return result
+
+ def run_inference(
+ self,
+ batch: Sequence[Dict[str, List[str]]],
+ model: ModelT,
+ inference_args: Optional[Dict[str, Any]] = None,
+ ) -> List[Dict[str, Union[List[float], List[str:
+"""
+Runs inference on a batch of text inputs. The inputs are expected to be
+a list of dicts. Each dict should have the same keys, and the shape
+should be of the same size for a single key across the batch.
+"""
+self._validate_batch(batch)
+dict_batch = _convert_list_of_dicts_to_dict_of_lists(list_of_dicts=batch)
+transformed_ba
(beam) branch master updated: Add ApplyBucketsWithInterpolation TFTransform (#31291)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 06e103d87e8 Add ApplyBucketsWithInterpolation TFTransform (#31291)
06e103d87e8 is described below
commit 06e103d87e8ac883f606475dbadbefed4ba77c9a
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Wed May 29 14:23:57 2024 -0400
Add ApplyBucketsWithInterpolation TFTransform (#31291)
* Add ApplyBucketsWithInterpolation TFTransform
* Update sdks/python/apache_beam/ml/transforms/tft.py
Co-authored-by: tvalentyn
* add tft documentation link
* change docstring wording around bucket_boundaries
* Update sdks/python/apache_beam/ml/transforms/tft.py
Co-authored-by: tvalentyn
-
Co-authored-by: tvalentyn
---
sdks/python/apache_beam/ml/transforms/tft.py | 49 ---
sdks/python/apache_beam/ml/transforms/tft_test.py | 30 ++
2 files changed, 74 insertions(+), 5 deletions(-)
diff --git a/sdks/python/apache_beam/ml/transforms/tft.py
b/sdks/python/apache_beam/ml/transforms/tft.py
index 370043bc0d9..e2f02971e7c 100644
--- a/sdks/python/apache_beam/ml/transforms/tft.py
+++ b/sdks/python/apache_beam/ml/transforms/tft.py
@@ -337,16 +337,16 @@ class ApplyBuckets(TFTOperation):
name: Optional[str] = None):
"""
This functions is used to map the element to a positive index i for
-which bucket_boundaries[i-1] <= element < bucket_boundaries[i],
-if it exists. If input < bucket_boundaries[0], then element is
-mapped to 0. If element >= bucket_boundaries[-1], then element is
+which `bucket_boundaries[i-1] <= element < bucket_boundaries[i]`,
+if it exists. If `input < bucket_boundaries[0]`, then element is
+mapped to 0. If `element >= bucket_boundaries[-1]`, then element is
mapped to len(bucket_boundaries). NaNs are mapped to
len(bucket_boundaries).
Args:
columns: A list of column names to apply the transformation on.
- bucket_boundaries: A rank 2 Tensor or list representing the bucket
-boundaries sorted in ascending order.
+ bucket_boundaries: An iterable of ints or floats representing the bucket
+boundaries. Must be sorted in ascending order.
name: (Optional) A string that specifies the name of the operation.
"""
super().__init__(columns)
@@ -363,6 +363,45 @@ class ApplyBuckets(TFTOperation):
return output
+@register_input_dtype(float)
+class ApplyBucketsWithInterpolation(TFTOperation):
+ def __init__(
+ self,
+ columns: List[str],
+ bucket_boundaries: Iterable[Union[int, float]],
+ name: Optional[str] = None):
+"""Interpolates values within the provided buckets and then normalizes to
+[0, 1].
+
+Input values are bucketized based on the provided boundaries such that the
+input is mapped to a positive index i for which `bucket_boundaries[i-1] <=
+element < bucket_boundaries[i]`, if it exists. The values are then
+normalized to the range [0,1] within the bucket, with NaN values being
+mapped to 0.5.
+
+For more information, see:
+
https://www.tensorflow.org/tfx/transform/api_docs/python/tft/apply_buckets_with_interpolation
+
+Args:
+ columns: A list of column names to apply the transformation on.
+ bucket_boundaries: An iterable of ints or floats representing the bucket
+boundaries sorted in ascending order.
+ name: (Optional) A string that specifies the name of the operation.
+"""
+super().__init__(columns)
+self.bucket_boundaries = [bucket_boundaries]
+self.name = name
+
+ def apply_transform(
+ self, data: common_types.TensorType,
+ output_column_name: str) -> Dict[str, common_types.TensorType]:
+output = {
+output_column_name: tft.apply_buckets_with_interpolation(
+x=data, bucket_boundaries=self.bucket_boundaries, name=self.name)
+}
+return output
+
+
@register_input_dtype(float)
class Bucketize(TFTOperation):
def __init__(
diff --git a/sdks/python/apache_beam/ml/transforms/tft_test.py
b/sdks/python/apache_beam/ml/transforms/tft_test.py
index 5c42ecc012f..f5615e9d4ad 100644
--- a/sdks/python/apache_beam/ml/transforms/tft_test.py
+++ b/sdks/python/apache_beam/ml/transforms/tft_test.py
@@ -364,6 +364,36 @@ class ApplyBucketsTest(unittest.TestCase):
actual_output, equal_to(expected_output, equals_fn=np.array_equal))
+class ApplyBucketsWithInterpolationTest(unittest.TestCase):
+ def setUp(self) -> None:
+self.artifact_location = tempfile.mkdtemp()
+
+ def tearDown(self):
+shutil.rmtree(self.artifact_location)
+
+ @parameterize
(beam) branch master updated: [Go SDK] Make it clearer that timers and data don't interact negatively in element batches. (#31319)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 0e734fe1c38 [Go SDK] Make it clearer that timers and data don't
interact negatively in element batches. (#31319)
0e734fe1c38 is described below
commit 0e734fe1c383b09096988b80551843374776c18f
Author: Robert Burke
AuthorDate: Tue May 21 07:11:07 2024 -0700
[Go SDK] Make it clearer that timers and data don't interact negatively in
element batches. (#31319)
* Don't swallow data errors on timer errors.
* reset error in case of timer handling
* Don't drop timers on splits.
-
Co-authored-by: lostluck <13907733+lostl...@users.noreply.github.com>
---
sdks/go/pkg/beam/core/runtime/exec/datasource.go | 33 +++-
1 file changed, 15 insertions(+), 18 deletions(-)
diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasource.go
b/sdks/go/pkg/beam/core/runtime/exec/datasource.go
index 674de44cf35..156d9565379 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/datasource.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/datasource.go
@@ -107,15 +107,10 @@ func (n *DataSource) StartBundle(ctx context.Context, id
string, data DataContex
return n.Out.StartBundle(ctx, id, data)
}
-// errSplitSuccess is a marker error to indicate we've reached the split index.
-// Akin to io.EOF.
-var errSplitSuccess = errors.New("split index reached")
-
// process handles converting elements from the data source to timers.
//
// The data and timer callback functions must return an io.EOF if the reader
terminates to signal that an additional
-// buffer is desired. On successful splits, [splitSuccess] must be returned to
indicate that the
-// PTransform is done processing data for this instruction.
+// buffer is desired.
func (n *DataSource) process(ctx context.Context, data func(bcr
*byteCountReader, ptransformID string) error, timer func(bcr *byteCountReader,
ptransformID, timerFamilyID string) error) error {
// The SID contains this instruction's expected data processing
transform (this one).
elms, err := n.source.OpenElementChan(ctx, n.SID,
maps.Keys(n.OnTimerTransforms))
@@ -129,7 +124,6 @@ func (n *DataSource) process(ctx context.Context, data
func(bcr *byteCountReader
var byteCount int
bcr := byteCountReader{reader: , count: }
- splitPrimaryComplete := map[string]bool{}
for {
var err error
select {
@@ -138,24 +132,21 @@ func (n *DataSource) process(ctx context.Context, data
func(bcr *byteCountReader
if !ok {
return nil
}
- if splitPrimaryComplete[e.PtransformID] {
- continue
- }
if len(e.Data) > 0 {
r.Reset(e.Data)
err = data(, e.PtransformID)
}
+ if err != nil && err != io.EOF {
+ return errors.Wrapf(err, "source failed
processing data")
+ }
+ // Process any simultaneously sent timers.
+ // If the data channel has split though
if len(e.Timers) > 0 {
r.Reset(e.Timers)
err = timer(, e.PtransformID,
e.TimerFamilyID)
}
-
- if err == errSplitSuccess {
- // Returning splitSuccess means we've split,
and aren't consuming the remaining buffer.
- // We mark the PTransform done to ignore
further data.
- splitPrimaryComplete[e.PtransformID] = true
- } else if err != nil && err != io.EOF {
- return errors.Wrap(err, "source failed")
+ if err != nil && err != io.EOF {
+ return errors.Wrap(err, "source failed
processing timers")
}
// io.EOF means the reader successfully drained.
// We're ready for a new buffer.
@@ -210,8 +201,13 @@ func (n *DataSource) Process(ctx context.Context)
([]*Checkpoint, error) {
cp = MakeElementDecoder(c)
}
+ hasSplit := map[string]bool{}
var checkpoints []*Checkpoint
err := n.process(ctx, func(bcr *byteCountReader, ptransformID string)
error {
+ // Check if this transform has already successfully, and if so,
skip reading and decoding of the elements in the buffer.
+
(beam) branch master updated: Implement DeduplicateTensorPerRow in MLTransform (#31307)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 118c51404d9 Implement DeduplicateTensorPerRow in MLTransform (#31307)
118c51404d9 is described below
commit 118c51404d93132d4a0cae135f00ff68d4fd84bf
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Thu May 16 10:29:04 2024 -0400
Implement DeduplicateTensorPerRow in MLTransform (#31307)
---
sdks/python/apache_beam/ml/transforms/tft.py | 22
sdks/python/apache_beam/ml/transforms/tft_test.py | 62 +++
2 files changed, 84 insertions(+)
diff --git a/sdks/python/apache_beam/ml/transforms/tft.py
b/sdks/python/apache_beam/ml/transforms/tft.py
index 550dbedbc7b..370043bc0d9 100644
--- a/sdks/python/apache_beam/ml/transforms/tft.py
+++ b/sdks/python/apache_beam/ml/transforms/tft.py
@@ -681,3 +681,25 @@ class HashStrings(TFTOperation):
name=self.name)
}
return output_dict
+
+
+@register_input_dtype(str)
+class DeduplicateTensorPerRow(TFTOperation):
+ def __init__(self, columns: List[str], name: Optional[str] = None):
+""" Deduplicates each row (0th dimension) of the provided tensor.
+
+Args:
+ columns: A list of the columns to apply the transformation on.
+ name: optional. A name for this operation.
+"""
+self.name = name
+super().__init__(columns)
+
+ def apply_transform(
+ self, data: common_types.TensorType,
+ output_col_name: str) -> Dict[str, common_types.TensorType]:
+output_dict = {
+output_col_name: tft.deduplicate_tensor_per_row(
+input_tensor=data, name=self.name)
+}
+return output_dict
diff --git a/sdks/python/apache_beam/ml/transforms/tft_test.py
b/sdks/python/apache_beam/ml/transforms/tft_test.py
index 6763032a8eb..5c42ecc012f 100644
--- a/sdks/python/apache_beam/ml/transforms/tft_test.py
+++ b/sdks/python/apache_beam/ml/transforms/tft_test.py
@@ -1009,5 +1009,67 @@ class HashWordsTest(unittest.TestCase):
assert_that(result, equal_to(expected_values, equals_fn=np.array_equal))
+class DeduplicateTensorPerRowTest(unittest.TestCase):
+ def setUp(self) -> None:
+self.artifact_location = tempfile.mkdtemp()
+
+ def tearDown(self):
+shutil.rmtree(self.artifact_location)
+
+ def test_deduplicate(self):
+values = [{
+'x': [b'a', b'b', b'a', b'b'],
+}, {
+'x': [b'b', b'c', b'b', b'c']
+}]
+
+expected_output = [np.array([b'a', b'b']), np.array([b'b', b'c'])]
+with beam.Pipeline() as p:
+ list_result = (
+ p
+ | "listCreate" >> beam.Create(values)
+ | "listMLTransform" >> base.MLTransform(
+ write_artifact_location=self.artifact_location).with_transform(
+ tft.DeduplicateTensorPerRow(columns=['x'])))
+ result = (list_result | beam.Map(lambda x: x.x))
+ assert_that(result, equal_to(expected_output, equals_fn=np.array_equal))
+
+ def test_deduplicate_no_op(self):
+values = [{
+'x': [b'a', b'b'],
+}, {
+'x': [b'c', b'd']
+}]
+
+expected_output = [np.array([b'a', b'b']), np.array([b'c', b'd'])]
+with beam.Pipeline() as p:
+ list_result = (
+ p
+ | "listCreate" >> beam.Create(values)
+ | "listMLTransform" >> base.MLTransform(
+ write_artifact_location=self.artifact_location).with_transform(
+ tft.DeduplicateTensorPerRow(columns=['x'])))
+ result = (list_result | beam.Map(lambda x: x.x))
+ assert_that(result, equal_to(expected_output, equals_fn=np.array_equal))
+
+ def test_deduplicate_different_output_sizes(self):
+values = [{
+'x': [b'a', b'b', b'a', b'b'],
+}, {
+'x': [b'c', b'a', b'd', b'd']
+}]
+
+expected_output = [np.array([b'a', b'b']), np.array([b'c', b'a', b'd'])]
+with beam.Pipeline() as p:
+ list_result = (
+ p
+ | "listCreate" >> beam.Create(values)
+ | "listMLTransform" >> base.MLTransform(
+ write_artifact_location=self.artifact_location).with_transform(
+ tft.DeduplicateTensorPerRow(columns=['x'])))
+ result = (list_result | beam.Map(lambda x: x.x))
+ assert_that(result, equal_to(expected_output, equals_fn=np.array_equal))
+
+
if __name__ == '__main__':
unittest.main()
(beam) branch master updated: Implement the hash_words TFT operation (#31249)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new c230655a1e5 Implement the hash_words TFT operation (#31249)
c230655a1e5 is described below
commit c230655a1e599e25a69b77d8a85fad45e5fc7587
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Mon May 13 14:22:25 2024 -0400
Implement the hash_words TFT operation (#31249)
* Implement the hash-words TFT operation
* linting
* Update sdks/python/apache_beam/ml/transforms/tft.py
Co-authored-by: tvalentyn
* tighten key type hint
* formatting
-
Co-authored-by: tvalentyn
---
sdks/python/apache_beam/ml/transforms/tft.py | 44 +
sdks/python/apache_beam/ml/transforms/tft_test.py | 77 +++
2 files changed, 121 insertions(+)
diff --git a/sdks/python/apache_beam/ml/transforms/tft.py
b/sdks/python/apache_beam/ml/transforms/tft.py
index 9b02cf8b75c..550dbedbc7b 100644
--- a/sdks/python/apache_beam/ml/transforms/tft.py
+++ b/sdks/python/apache_beam/ml/transforms/tft.py
@@ -637,3 +637,47 @@ class BagOfWords(TFTOperation):
def count_unique_words(
data: tf.SparseTensor, output_vocab_name: Optional[str]) -> None:
tft.count_per_key(data, key_vocabulary_filename=output_vocab_name)
+
+
+@register_input_dtype(str)
+class HashStrings(TFTOperation):
+ def __init__(
+ self,
+ columns: List[str],
+ hash_buckets: int,
+ key: Optional[Tuple[int, int]] = None,
+ name: Optional[str] = None):
+'''Hashes strings into the provided number of buckets.
+
+Args:
+ columns: A list of the column names to apply the transformation on.
+ hash_buckets: the number of buckets to hash the strings into.
+ key: optional. An array of two Python `uint64`. If passed, output will be
+a deterministic function of `strings` and `key`. Note that hashing will
+be slower if this value is specified.
+ name: optional. A name for this operation.
+
+Raises:
+ ValueError if `hash_buckets` is not a positive and non-zero integer.
+'''
+self.hash_buckets = hash_buckets
+self.key = key
+self.name = name
+
+if hash_buckets < 1:
+ raise ValueError(
+ 'number of hash buckets must be positive, got ', hash_buckets)
+
+super().__init__(columns)
+
+ def apply_transform(
+ self, data: common_types.TensorType,
+ output_col_name: str) -> Dict[str, common_types.TensorType]:
+output_dict = {
+output_col_name: tft.hash_strings(
+strings=data,
+hash_buckets=self.hash_buckets,
+key=self.key,
+name=self.name)
+}
+return output_dict
diff --git a/sdks/python/apache_beam/ml/transforms/tft_test.py
b/sdks/python/apache_beam/ml/transforms/tft_test.py
index ed7c301f3b8..6763032a8eb 100644
--- a/sdks/python/apache_beam/ml/transforms/tft_test.py
+++ b/sdks/python/apache_beam/ml/transforms/tft_test.py
@@ -932,5 +932,82 @@ class BagOfWordsTest(unittest.TestCase):
self.assertEqual(expected_data, actual_data)
+class HashWordsTest(unittest.TestCase):
+ def setUp(self) -> None:
+self.artifact_location = tempfile.mkdtemp()
+
+ def tearDown(self):
+shutil.rmtree(self.artifact_location)
+
+ def test_single_bucket(self):
+list_data = [{'x': 'this is a test string'}]
+expected_values = [np.array([0])]
+with beam.Pipeline() as p:
+ list_result = (
+ p
+ | "listCreate" >> beam.Create(list_data)
+ | "listMLTransform" >> base.MLTransform(
+ write_artifact_location=self.artifact_location).with_transform(
+ tft.HashStrings(columns=['x'], hash_buckets=1)))
+ result = (list_result | beam.Map(lambda x: x.x))
+ assert_that(result, equal_to(expected_values, equals_fn=np.array_equal))
+
+ def test_multi_bucket_one_string(self):
+list_data = [{'x': 'this is a test string'}]
+expected_values = [np.array([1])]
+with beam.Pipeline() as p:
+ list_result = (
+ p
+ | "listCreate" >> beam.Create(list_data)
+ | "listMLTransform" >> base.MLTransform(
+ write_artifact_location=self.artifact_location).with_transform(
+ tft.HashStrings(columns=['x'], hash_buckets=2)))
+ result = (list_result | beam.Map(lambda x: x.x))
+ assert_that(result, equal_to(expected_values, equals_fn=np.array_equal))
+
+ def test_one_bucket_multi_string(self):
+list_data = [{'x': ['these', 'are', 'test', 'strings']}]
+expected_values = [np.array([0, 0, 0, 0])]
+with beam.Pipeline() as p:
+ list_result = (
+ p
+ | "l
(beam) branch master updated: Avoid oversizing batch sizes with size estimation function (#31228)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 2d53926542f Avoid oversizing batch sizes with size estimation function
(#31228)
2d53926542f is described below
commit 2d53926542f82a8b955eb541f13475f9bef091a7
Author: Danny McCormick
AuthorDate: Fri May 10 10:12:17 2024 -0400
Avoid oversizing batch sizes with size estimation function (#31228)
* Avoid oversizing batch sizes with size estimation function
* lint
---
sdks/python/apache_beam/transforms/util.py | 18 --
sdks/python/apache_beam/transforms/util_test.py | 33 ++---
2 files changed, 40 insertions(+), 11 deletions(-)
diff --git a/sdks/python/apache_beam/transforms/util.py
b/sdks/python/apache_beam/transforms/util.py
index edf79b7c798..750d98f0789 100644
--- a/sdks/python/apache_beam/transforms/util.py
+++ b/sdks/python/apache_beam/transforms/util.py
@@ -579,14 +579,15 @@ class _GlobalWindowsBatchingDoFn(DoFn):
self._batch_size_estimator.ignore_next_timing()
def process(self, element):
-self._batch.append(element)
-self._running_batch_size += self._element_size_fn(element)
-if self._running_batch_size >= self._target_batch_size:
+element_size = self._element_size_fn(element)
+if self._running_batch_size + element_size > self._target_batch_size:
with self._batch_size_estimator.record_time(self._running_batch_size):
yield window.GlobalWindows.windowed_value_at_end_of_window(self._batch)
self._batch = []
self._running_batch_size = 0
self._target_batch_size = self._batch_size_estimator.next_batch_size()
+self._batch.append(element)
+self._running_batch_size += element_size
def finish_bundle(self):
if self._batch:
@@ -621,15 +622,18 @@ class _WindowAwareBatchingDoFn(DoFn):
def process(self, element, window=DoFn.WindowParam):
batch = self._batches[window]
-batch.elements.append(element)
-batch.size += self._element_size_fn(element)
-if batch.size >= self._target_batch_size:
+element_size = self._element_size_fn(element)
+if batch.size + element_size > self._target_batch_size:
with self._batch_size_estimator.record_time(batch.size):
yield windowed_value.WindowedValue(
batch.elements, window.max_timestamp(), (window, ))
del self._batches[window]
self._target_batch_size = self._batch_size_estimator.next_batch_size()
-elif len(self._batches) > self._MAX_LIVE_WINDOWS:
+
+self._batches[window].elements.append(element)
+self._batches[window].size += element_size
+
+if len(self._batches) > self._MAX_LIVE_WINDOWS:
window, batch = max(
self._batches.items(),
key=lambda window_batch: window_batch[1].size)
diff --git a/sdks/python/apache_beam/transforms/util_test.py
b/sdks/python/apache_beam/transforms/util_test.py
index 53898d57998..74d9f438a5d 100644
--- a/sdks/python/apache_beam/transforms/util_test.py
+++ b/sdks/python/apache_beam/transforms/util_test.py
@@ -299,15 +299,40 @@ class BatchElementsTest(unittest.TestCase):
res = (
p
| beam.Create([
- 'a', 'a', 'aa', # First batch.
- 'aa', 'a', # Second batch.
- 'a', 'aaa', 'a', 'a' # Third batch.
+ 'a', 'a',# First batch.
+ 'aa',# Second batch.
+ 'a', 'a',# Third batch.
+ 'a', 'aaa', 'a', 'a' # Fourth batch.
], reshuffle=False)
| util.BatchElements(
min_batch_size=10, max_batch_size=10, element_size_fn=len)
| beam.Map(lambda batch: ''.join(batch))
| beam.Map(len))
- assert_that(res, equal_to([12, 11, 10]))
+ assert_that(res, equal_to([2, 10, 10, 10]))
+
+ def test_sized_windowed_batches(self):
+# Assumes a single bundle, in order...
+with TestPipeline() as p:
+ res = (
+ p
+ | beam.Create(range(1, 8), reshuffle=False)
+ | beam.Map(lambda t: window.TimestampedValue('a' * t, t))
+ | beam.WindowInto(window.FixedWindows(3))
+ | util.BatchElements(
+ min_batch_size=11,
+ max_batch_size=11,
+ element_size_fn=len,
+ clock=FakeClock())
+ | beam.Map(lambda batch: ''.join(batch)))
+ assert_that(
+ res,
+ equal_to([
+ 'a' * (1+2), # Elements in [1, 3)
+ 'a' * (3+4), # Elements in [3, 6)
+ 'a' * 5,
+ 'a' * 6, # Elements in [6, 9)
+ 'a' * 7,
+ ]))
def test_target_duration(self):
clock = FakeClock()
(beam) branch master updated (20409b5204d -> faaa68c1a1c)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from 20409b5204d Fix python_test broke by #31199 (#31221) add faaa68c1a1c Fix lint warning: JobId vs JobID (#31197) No new revisions were added by this update. Summary of changes: sdks/go/pkg/beam/runners/prism/internal/web/web.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
(beam) branch master updated: Bump jinja2 from 2.11.3 to 3.1.4 in /.test-infra/jenkins/metrics_report (#31191)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new a22ee682fa8 Bump jinja2 from 2.11.3 to 3.1.4 in /.test-infra/jenkins/metrics_report (#31191) a22ee682fa8 is described below commit a22ee682fa87a126a1efd2f3ddb3c33913506654 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Mon May 6 15:08:39 2024 -0400 Bump jinja2 from 2.11.3 to 3.1.4 in /.test-infra/jenkins/metrics_report (#31191) Bumps [jinja2](https://github.com/pallets/jinja) from 2.11.3 to 3.1.4. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/2.11.3...3.1.4) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .test-infra/jenkins/metrics_report/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/metrics_report/requirements.txt b/.test-infra/jenkins/metrics_report/requirements.txt index 4716e65ddb9..e40f8d92982 100644 --- a/.test-infra/jenkins/metrics_report/requirements.txt +++ b/.test-infra/jenkins/metrics_report/requirements.txt @@ -20,5 +20,5 @@ # https://github.com/aws/aws-sam-cli/issues/3661 markupsafe==2.0.1 influxdb==5.3.0 -Jinja2==2.11.3 +Jinja2==3.1.4 prettytable==0.7.2
(beam) branch master updated: Pass location for Vertex endpoint retrieval (#31171)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 7dbdbdfbad4 Pass location for Vertex endpoint retrieval (#31171)
7dbdbdfbad4 is described below
commit 7dbdbdfbad45f50eab1b1de7f5589b3f28903934
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Fri May 3 12:50:07 2024 -0400
Pass location for Vertex endpoint retrieval (#31171)
---
.../apache_beam/ml/inference/vertex_ai_inference.py | 15 ++-
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py
b/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py
index 95660441a84..4c4163accfb 100644
--- a/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py
+++ b/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py
@@ -133,9 +133,11 @@ class VertexAIModelHandlerJSON(ModelHandler[Any,
# Check for liveness here but don't try to actually store the endpoint
# in the class yet
self.endpoint_name = endpoint_id
+self.location = location
self.is_private = private
-_ = self._retrieve_endpoint(self.endpoint_name, self.is_private)
+_ = self._retrieve_endpoint(
+self.endpoint_name, self.location, self.is_private)
# Configure AdaptiveThrottler and throttling metrics for client-side
# throttling behavior.
@@ -147,7 +149,8 @@ class VertexAIModelHandlerJSON(ModelHandler[Any,
window_ms=1, bucket_ms=1, overload_ratio=2)
def _retrieve_endpoint(
- self, endpoint_id: str, is_private: bool) -> aiplatform.Endpoint:
+ self, endpoint_id: str, location: str,
+ is_private: bool) -> aiplatform.Endpoint:
"""Retrieves an AI Platform endpoint and queries it for liveness/deployed
models.
@@ -162,10 +165,11 @@ class VertexAIModelHandlerJSON(ModelHandler[Any,
"""
if is_private:
endpoint: aiplatform.Endpoint = aiplatform.PrivateEndpoint(
- endpoint_name=endpoint_id)
+ endpoint_name=endpoint_id, location=location)
LOGGER.debug("Treating endpoint %s as private", endpoint_id)
else:
- endpoint = aiplatform.Endpoint(endpoint_name=endpoint_id)
+ endpoint = aiplatform.Endpoint(
+ endpoint_name=endpoint_id, location=location)
LOGGER.debug("Treating endpoint %s as public", endpoint_id)
try:
@@ -185,7 +189,8 @@ class VertexAIModelHandlerJSON(ModelHandler[Any,
"""
# Check to make sure the endpoint is still active since pipeline
# construction time
-ep = self._retrieve_endpoint(self.endpoint_name, self.is_private)
+ep = self._retrieve_endpoint(
+self.endpoint_name, self.location, self.is_private)
return ep
@retry.with_exponential_backoff(
(beam) branch master updated: Pin tensorflow version in notebook (#31145)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 44e2abea788 Pin tensorflow version in notebook (#31145) 44e2abea788 is described below commit 44e2abea7887cd692ef7e5623c632442858c51ba Author: Danny McCormick AuthorDate: Wed May 1 13:57:03 2024 -0400 Pin tensorflow version in notebook (#31145) --- examples/notebooks/beam-ml/automatic_model_refresh.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/notebooks/beam-ml/automatic_model_refresh.ipynb b/examples/notebooks/beam-ml/automatic_model_refresh.ipynb index 54fb16b6ef0..5b5d2ed484c 100644 --- a/examples/notebooks/beam-ml/automatic_model_refresh.ipynb +++ b/examples/notebooks/beam-ml/automatic_model_refresh.ipynb @@ -131,7 +131,7 @@ }], "source": [ "!pip install apache_beam[gcp]>=2.46.0 --quiet\n", -"!pip install tensorflow --quiet\n", +"!pip install tensorflow==2.15.0 --quiet\n", "!pip install tensorflow_hub --quiet" ] }, @@ -272,7 +272,7 @@ "cell_type": "code", "source": [ "# In a requirements file, define the dependencies required for the pipeline.\n", -"!printf 'tensorflow>=2.12.0\\ntensorflow_hub>=0.10.0\\nPillow>=9.0.0' > ./requirements.txt\n", +"!printf 'tensorflow==2.15.0\\ntensorflow_hub>=0.10.0\\nPillow>=9.0.0' > ./requirements.txt\n", "# Install the pipeline dependencies on Dataflow.\n", "options.view_as(SetupOptions).requirements_file = './requirements.txt'" ],
(beam) branch master updated (de665b77d9f -> ba7ea856ffe)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from de665b77d9f [YAML] - Normalize YAML PubSub format (#31068) add ba7ea856ffe Revert "Change type for UnboundedReaderMaxReadTimeSec (#31037)" (#31082) No new revisions were added by this update. Summary of changes: .../runners/dataflow/options/DataflowPipelineDebugOptions.java | 9 - .../apache/beam/runners/dataflow/worker/WorkerCustomSources.java | 4 +--- .../beam/runners/dataflow/worker/WorkerCustomSourcesTest.java| 7 +++ 3 files changed, 8 insertions(+), 12 deletions(-)
(beam) 01/01: Revert "Change type for UnboundedReaderMaxReadTimeSec (#31037)"
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch
revert-31037-readerMaxReadTimeSec_as_double
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 5c8f548c944a9bbc32e07a13eefaac5493b26808
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Tue Apr 23 10:42:04 2024 -0400
Revert "Change type for UnboundedReaderMaxReadTimeSec (#31037)"
This reverts commit bb310e7e90720b620f1089574f1656ca84a3656d.
---
.../runners/dataflow/options/DataflowPipelineDebugOptions.java | 9 -
.../apache/beam/runners/dataflow/worker/WorkerCustomSources.java | 4 +---
.../beam/runners/dataflow/worker/WorkerCustomSourcesTest.java| 7 +++
3 files changed, 8 insertions(+), 12 deletions(-)
diff --git
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
index 3f6c47ece68..30496dec296 100644
---
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
+++
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
@@ -218,12 +218,11 @@ public interface DataflowPipelineDebugOptions
/** The max amount of time an UnboundedReader is consumed before
checkpointing. */
@Description(
- "The max amount of time before an UnboundedReader is consumed before
checkpointing, "
- + "in seconds. Duration can be set to fractions of seconds. ")
- @Default.Double(10.0)
- double getUnboundedReaderMaxReadTimeSec();
+ "The max amount of time before an UnboundedReader is consumed before
checkpointing, in seconds.")
+ @Default.Integer(10)
+ Integer getUnboundedReaderMaxReadTimeSec();
- void setUnboundedReaderMaxReadTimeSec(double value);
+ void setUnboundedReaderMaxReadTimeSec(Integer value);
/** The max elements read from an UnboundedReader before checkpointing. */
@Description("The max elements read from an UnboundedReader before
checkpointing. ")
diff --git
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
index a8e358f19e0..8c086016ee9 100644
---
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
+++
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
@@ -798,9 +798,7 @@ public class WorkerCustomSources {
DataflowPipelineDebugOptions debugOptions =
options.as(DataflowPipelineDebugOptions.class);
this.endTime =
Instant.now()
- .plus(
- Duration.millis(
- (long) (debugOptions.getUnboundedReaderMaxReadTimeSec()
* 1000L)));
+
.plus(Duration.standardSeconds(debugOptions.getUnboundedReaderMaxReadTimeSec()));
this.maxElems = debugOptions.getUnboundedReaderMaxElements();
this.backoffFactory =
FluentBackoff.DEFAULT
diff --git
a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
index 261567930fe..d451ec093f7 100644
---
a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
+++
b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
@@ -598,7 +598,6 @@ public class WorkerCustomSourcesTest {
int maxElements = 10;
DataflowPipelineDebugOptions debugOptions =
options.as(DataflowPipelineDebugOptions.class);
debugOptions.setUnboundedReaderMaxElements(maxElements);
-debugOptions.setUnboundedReaderMaxReadTimeSec(10);
ByteString state = ByteString.EMPTY;
for (int i = 0; i < 10 * maxElements;
@@ -646,10 +645,10 @@ public class WorkerCustomSourcesTest {
numReadOnThisIteration++;
}
Instant afterReading = Instant.now();
- double maxReadSec = debugOptions.getUnboundedReaderMaxReadTimeSec();
+ long maxReadSec = debugOptions.getUnboundedReaderMaxReadTimeSec();
assertThat(
- new Duration(beforeReading, afterReading).getMillis(),
- lessThanOrEqualTo((long) ((maxReadSec + 1) * 1000L)));
+ new Duration(beforeReading, afterReading).getStandardSeconds(),
+ lessThanOrEqualTo(maxReadSec + 1));
assert
(beam) branch revert-31037-readerMaxReadTimeSec_as_double created (now 5c8f548c944)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch revert-31037-readerMaxReadTimeSec_as_double in repository https://gitbox.apache.org/repos/asf/beam.git at 5c8f548c944 Revert "Change type for UnboundedReaderMaxReadTimeSec (#31037)" This branch includes the following new commits: new 5c8f548c944 Revert "Change type for UnboundedReaderMaxReadTimeSec (#31037)" The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(beam) branch master updated: Add ScaleToGaussian TFTransform (#31026)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new fd1003034ff Add ScaleToGaussian TFTransform (#31026)
fd1003034ff is described below
commit fd1003034ff1232165f963f9aa8b0a128b7ff11e
Author: Jack McCluskey <34928439+jrmcclus...@users.noreply.github.com>
AuthorDate: Fri Apr 19 11:30:41 2024 -0400
Add ScaleToGaussian TFTransform (#31026)
* Add ScaleToGaussian TFTransform
* address notes
* formatting
---
sdks/python/apache_beam/ml/transforms/tft.py | 37 +++
sdks/python/apache_beam/ml/transforms/tft_test.py | 126 ++
2 files changed, 163 insertions(+)
diff --git a/sdks/python/apache_beam/ml/transforms/tft.py
b/sdks/python/apache_beam/ml/transforms/tft.py
index cf44ee9ca0f..9b02cf8b75c 100644
--- a/sdks/python/apache_beam/ml/transforms/tft.py
+++ b/sdks/python/apache_beam/ml/transforms/tft.py
@@ -291,6 +291,43 @@ class ScaleTo01(TFTOperation):
return output_dict
+@register_input_dtype(float)
+class ScaleToGaussian(TFTOperation):
+ def __init__(
+ self,
+ columns: List[str],
+ elementwise: bool = False,
+ name: Optional[str] = None):
+"""
+This operation scales the given input column values to an approximately
+normal distribution with mean 0 and variance of 1. The Gaussian
+transformation is only applied if the column has long tails;
+otherwise, the transformation is the same as normalizing to z scores.
+
+For more information, see:
+
https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_to_gaussian
+
+Args:
+ columns: A list of column names to apply the transformation on.
+ elementwise: If True, the transformation is applied elementwise.
+Otherwise, the transformation is applied on the entire column.
+ name: A name for the operation (optional).
+
+"""
+super().__init__(columns)
+self.elementwise = elementwise
+self.name = name
+
+ def apply_transform(
+ self, data: common_types.TensorType,
+ output_column_name: str) -> Dict[str, common_types.TensorType]:
+output_dict = {
+output_column_name: tft.scale_to_gaussian(
+x=data, elementwise=self.elementwise, name=self.name)
+}
+return output_dict
+
+
@register_input_dtype(float)
class ApplyBuckets(TFTOperation):
def __init__(
diff --git a/sdks/python/apache_beam/ml/transforms/tft_test.py
b/sdks/python/apache_beam/ml/transforms/tft_test.py
index 558b4ede2ec..ed7c301f3b8 100644
--- a/sdks/python/apache_beam/ml/transforms/tft_test.py
+++ b/sdks/python/apache_beam/ml/transforms/tft_test.py
@@ -155,6 +155,132 @@ class ScaleTo01Test(unittest.TestCase):
actual_output, equal_to(expected_output, equals_fn=np.array_equal))
+class ScaleToGaussianTest(unittest.TestCase):
+ def setUp(self) -> None:
+self.artifact_location = tempfile.mkdtemp()
+
+ def tearDown(self):
+shutil.rmtree(self.artifact_location)
+
+ def test_gaussian_list_uniform_distribution(self):
+list_data = [{'x': [1, 2, 3]}, {'x': [4, 5, 6]}]
+with beam.Pipeline() as p:
+ list_result = (
+ p
+ | "listCreate" >> beam.Create(list_data)
+ | "listMLTransform" >> base.MLTransform(
+ write_artifact_location=self.artifact_location).with_transform(
+ tft.ScaleToGaussian(columns=['x'])))
+
+ expected_data = [
+ np.array([-1.46385, -0.87831, -0.29277], dtype=np.float32),
+ np.array([0.29277, 0.87831, 1.46385], dtype=np.float32)
+ ]
+ actual_data = (list_result | beam.Map(lambda x: x.x))
+ assert_that(
+ actual_data, equal_to(expected_data, equals_fn=np.array_equal))
+
+ def test_gaussian_list_skewed_distribution(self):
+list_data = [{'x': [1, 2, 4]}, {'x': [8, 16, 32]}]
+with beam.Pipeline() as p:
+ list_result = (
+ p
+ | "listCreate" >> beam.Create(list_data)
+ | "listMLTransform" >> base.MLTransform(
+ write_artifact_location=self.artifact_location).with_transform(
+ tft.ScaleToGaussian(columns=['x'])))
+
+ expected_data = [
+ np.array([-0.87733847, -0.78498703, -0.6002842], dtype=np.float32),
+ np.array([-0.23087855, 0.5079328, 1.985], dtype=np.float32)
+ ]
+ actual_data = (list_result | beam.Map(lambda x: x.x))
+ assert_that(actual_data, equal_to(expected_data, equals_fn=np.allclose))
+
+ def test_gaussian_uniform(self):
+data = [
+{
+'x': 1
+},
+{
+'x': 2
+},
+{
+'x': 3
+},
+{
+'x': 4
+},
+{
+
(beam) branch master updated (b69e8c615af -> 76c77cd28ae)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/beam.git from b69e8c615af Updates Python Dev container used by Dataflow (#31029) add 76c77cd28ae Fix typo in count_unique_words() (#31023) No new revisions were added by this update. Summary of changes: sdks/python/apache_beam/ml/transforms/tft.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
(beam) branch master updated: Change type for UnboundedReaderMaxReadTimeSec (#31037)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new bb310e7e907 Change type for UnboundedReaderMaxReadTimeSec (#31037)
bb310e7e907 is described below
commit bb310e7e90720b620f1089574f1656ca84a3656d
Author: Radosław Stankiewicz
AuthorDate: Thu Apr 18 21:50:05 2024 +0200
Change type for UnboundedReaderMaxReadTimeSec (#31037)
* add ms part for UnboundedReader checkpointing
* typo
* spotless
* spotless
* spotless
* [IntLongMath] Expression of type int may overflow before being assigned
to a long
* readerMaxReadTime sec as double
* readerMaxReadTime sec as double
* readerMaxReadTime sec as double
* readerMaxReadTime sec as double
* spotless
---
.../runners/dataflow/options/DataflowPipelineDebugOptions.java | 9 +
.../apache/beam/runners/dataflow/worker/WorkerCustomSources.java | 4 +++-
.../beam/runners/dataflow/worker/WorkerCustomSourcesTest.java| 7 ---
3 files changed, 12 insertions(+), 8 deletions(-)
diff --git
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
index 30496dec296..3f6c47ece68 100644
---
a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
+++
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
@@ -218,11 +218,12 @@ public interface DataflowPipelineDebugOptions
/** The max amount of time an UnboundedReader is consumed before
checkpointing. */
@Description(
- "The max amount of time before an UnboundedReader is consumed before
checkpointing, in seconds.")
- @Default.Integer(10)
- Integer getUnboundedReaderMaxReadTimeSec();
+ "The max amount of time before an UnboundedReader is consumed before
checkpointing, "
+ + "in seconds. Duration can be set to fractions of seconds. ")
+ @Default.Double(10.0)
+ double getUnboundedReaderMaxReadTimeSec();
- void setUnboundedReaderMaxReadTimeSec(Integer value);
+ void setUnboundedReaderMaxReadTimeSec(double value);
/** The max elements read from an UnboundedReader before checkpointing. */
@Description("The max elements read from an UnboundedReader before
checkpointing. ")
diff --git
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
index 8c086016ee9..a8e358f19e0 100644
---
a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
+++
b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSources.java
@@ -798,7 +798,9 @@ public class WorkerCustomSources {
DataflowPipelineDebugOptions debugOptions =
options.as(DataflowPipelineDebugOptions.class);
this.endTime =
Instant.now()
-
.plus(Duration.standardSeconds(debugOptions.getUnboundedReaderMaxReadTimeSec()));
+ .plus(
+ Duration.millis(
+ (long) (debugOptions.getUnboundedReaderMaxReadTimeSec()
* 1000L)));
this.maxElems = debugOptions.getUnboundedReaderMaxElements();
this.backoffFactory =
FluentBackoff.DEFAULT
diff --git
a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
index d451ec093f7..261567930fe 100644
---
a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
+++
b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java
@@ -598,6 +598,7 @@ public class WorkerCustomSourcesTest {
int maxElements = 10;
DataflowPipelineDebugOptions debugOptions =
options.as(DataflowPipelineDebugOptions.class);
debugOptions.setUnboundedReaderMaxElements(maxElements);
+debugOptions.setUnboundedReaderMaxReadTimeSec(10);
ByteString state = ByteString.EMPTY;
for (int i = 0; i < 10 * maxElements;
@@ -645,10 +646,10 @@ public class WorkerCustomSourcesTest {
numReadOnThisIteration++;
}
Instant afterReading = Instant.now();
- long maxReadSec = de
(beam) branch master updated: Bump transformers (#30922)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new 1a0591ea797 Bump transformers (#30922) 1a0591ea797 is described below commit 1a0591ea797440d2f4a6eb61fa7d87432856f872 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Fri Apr 12 13:53:56 2024 -0400 Bump transformers (#30922) Bumps [transformers](https://github.com/huggingface/transformers) from 4.30.0 to 4.38.0. - [Release notes](https://github.com/huggingface/transformers/releases) - [Commits](https://github.com/huggingface/transformers/compare/v4.30.0...v4.38.0) --- updated-dependencies: - dependency-name: transformers dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../examples/inference/online_clustering/clustering_pipeline/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/examples/inference/online_clustering/clustering_pipeline/setup.py b/sdks/python/apache_beam/examples/inference/online_clustering/clustering_pipeline/setup.py index 321128cce89..572763492f4 100644 --- a/sdks/python/apache_beam/examples/inference/online_clustering/clustering_pipeline/setup.py +++ b/sdks/python/apache_beam/examples/inference/online_clustering/clustering_pipeline/setup.py @@ -28,7 +28,7 @@ from setuptools import find_packages REQUIREMENTS = [ "apache-beam[gcp]==2.40.0", -"transformers==4.30.0", +"transformers==4.38.0", "torch==1.13.1", "scikit-learn==1.0.2", ]
(beam) branch master updated: Bump github.com/aws/smithy-go from 1.20.1 to 1.20.2 in /sdks (#30883)
This is an automated email from the ASF dual-hosted git repository. jrmccluskey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git The following commit(s) were added to refs/heads/master by this push: new f39a179fbf3 Bump github.com/aws/smithy-go from 1.20.1 to 1.20.2 in /sdks (#30883) f39a179fbf3 is described below commit f39a179fbf32dd57f425e73027fcb595a6595572 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> AuthorDate: Mon Apr 8 17:25:04 2024 -0400 Bump github.com/aws/smithy-go from 1.20.1 to 1.20.2 in /sdks (#30883) Bumps [github.com/aws/smithy-go](https://github.com/aws/smithy-go) from 1.20.1 to 1.20.2. - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.20.1...v1.20.2) --- updated-dependencies: - dependency-name: github.com/aws/smithy-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- sdks/go.mod | 2 +- sdks/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/go.mod b/sdks/go.mod index 05565b4df47..65d244b265c 100644 --- a/sdks/go.mod +++ b/sdks/go.mod @@ -35,7 +35,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.17.8 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.8 github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2 - github.com/aws/smithy-go v1.20.1 + github.com/aws/smithy-go v1.20.2 github.com/docker/go-connections v0.5.0 github.com/dustin/go-humanize v1.0.1 github.com/go-sql-driver/mysql v1.8.0 diff --git a/sdks/go.sum b/sdks/go.sum index d15eeac8589..4a51032d86b 100644 --- a/sdks/go.sum +++ b/sdks/go.sum @@ -128,8 +128,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.6.0/go.mod h1:q7o0j7d7HrJk/vr9uUt3BV github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3FajfLxqM5+tepvVXmxg= github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0= github.com/aws/smithy-go v1.6.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw= -github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= +github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
