This is an automated email from the ASF dual-hosted git repository.
hxb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 7e9be78974f [FLINK-25188][python][build] Support m1 chip.
7e9be78974f is described below
commit 7e9be78974f95e4eb3c8bb442564c81ea61c563e
Author: Ada Wong <r...@foxmail.com>
AuthorDate: Tue Jan 18 16:06:11 2022 +0800
[FLINK-25188][python][build] Support m1 chip.
This closes #18769.
---
NOTICE | 2 +-
flink-python/dev/dev-requirements.txt | 13 ++--
flink-python/dev/lint-python.sh | 6 ++
flink-python/lib/cloudpickle-1.2.2-src.zip | Bin 23105 -> 0 bytes
flink-python/lib/cloudpickle-2.0.0-src.zip | Bin 0 -> 24548 bytes
flink-python/pom.xml | 19 ++++--
.../pyflink/fn_execution/beam/beam_operations.py | 8 ++-
.../pyflink/table/tests/test_dependency.py | 2 +-
flink-python/setup.py | 10 +--
.../fnexecution/state/GrpcStateService.java | 6 +-
.../io/grpc/internal/SharedResourceHolder.java | 4 +-
.../beam/BeamDataStreamPythonFunctionRunner.java | 4 +-
.../python/beam/BeamPythonFunctionRunner.java | 8 +--
.../python/beam/state/BeamBagStateHandler.java | 2 +-
.../python/beam/state/BeamMapStateHandler.java | 2 +-
.../python/beam/state/BeamStateRequestHandler.java | 4 +-
.../flink/streaming/api/utils/ProtoUtils.java | 2 +-
.../python/beam/BeamTablePythonFunctionRunner.java | 2 +-
flink-python/src/main/resources/META-INF/NOTICE | 75 +++++++++++----------
.../PassThroughPythonAggregateFunctionRunner.java | 2 +-
.../PassThroughPythonScalarFunctionRunner.java | 2 +-
.../PassThroughPythonTableFunctionRunner.java | 2 +-
...ThroughStreamAggregatePythonFunctionRunner.java | 2 +-
...amGroupWindowAggregatePythonFunctionRunner.java | 2 +-
...ghStreamTableAggregatePythonFunctionRunner.java | 2 +-
pom.xml | 2 +-
tools/releasing/NOTICE-binary_PREAMBLE.txt | 2 +-
27 files changed, 102 insertions(+), 83 deletions(-)
diff --git a/NOTICE b/NOTICE
index 759a9f65adb..98237e26c68 100644
--- a/NOTICE
+++ b/NOTICE
@@ -17,7 +17,7 @@ See bundled license files for details.
This project bundles the following dependencies under the BSD license.
See bundled license files for details.
-- cloudpickle:1.2.2
+- cloudpickle:2.0.0
- net.sf.py4j:py4j:0.10.9.3
This project bundles the following dependencies under SIL OFL 1.1 license
(https://opensource.org/licenses/OFL-1.1).
diff --git a/flink-python/dev/dev-requirements.txt
b/flink-python/dev/dev-requirements.txt
index b061a67a20d..e5bad6014cc 100755
--- a/flink-python/dev/dev-requirements.txt
+++ b/flink-python/dev/dev-requirements.txt
@@ -14,17 +14,18 @@
# limitations under the License.
setuptools>=18.0
wheel
-apache-beam==2.27.0
+apache-beam==2.38.0
cython==0.29.24
py4j==0.10.9.3
python-dateutil==2.8.0
-cloudpickle==1.2.2
+cloudpickle==2.0.0
avro-python3>=1.8.1,!=1.9.2,<1.10.0
-pandas>=1.0,<1.2.0
-pyarrow>=0.15.1,<3.0.0
+pandas>=1.3.0
+pyarrow>=5.0.0
pytz>=2018.3
-numpy>=1.14.3,<1.20
+numpy>=1.21.4
fastavro>=0.21.4,<0.24
grpcio>=1.29.0,<2
grpcio-tools>=1.3.5,<=1.14.2
-pemja==0.1.4; python_version >= '3.7'
+pemja==0.1.5; python_version >= '3.7'
+httplib2>=0.8,<0.19.0
diff --git a/flink-python/dev/lint-python.sh b/flink-python/dev/lint-python.sh
index 09d986fb7f7..630a0505cf4 100755
--- a/flink-python/dev/lint-python.sh
+++ b/flink-python/dev/lint-python.sh
@@ -209,6 +209,12 @@ function install_miniconda() {
if [ ! -d "$CURRENT_DIR/.conda" ]; then
print_function "STEP" "installing conda..."
$CONDA_INSTALL_SH -b -p $CURRENT_DIR/.conda 2>&1 >/dev/null
+
+ # orjson depend on pip >= 20.3
+ print_function "STEP" "upgrade pip..."
+ $CURRENT_DIR/.conda/bin/python -m pip install --upgrade pip 2>&1
>/dev/null
+ print_function "STEP" "upgrade pip... [SUCCESS]"
+
if [ $? -ne 0 ]; then
echo "install miniconda failed"
exit $CONDA_INSTALL_STATUS
diff --git a/flink-python/lib/cloudpickle-1.2.2-src.zip
b/flink-python/lib/cloudpickle-1.2.2-src.zip
deleted file mode 100644
index 4d73a881f5e..00000000000
Binary files a/flink-python/lib/cloudpickle-1.2.2-src.zip and /dev/null differ
diff --git a/flink-python/lib/cloudpickle-2.0.0-src.zip
b/flink-python/lib/cloudpickle-2.0.0-src.zip
new file mode 100644
index 00000000000..ed416829018
Binary files /dev/null and b/flink-python/lib/cloudpickle-2.0.0-src.zip differ
diff --git a/flink-python/pom.xml b/flink-python/pom.xml
index 2783ecd70da..3e62acf94c6 100644
--- a/flink-python/pom.xml
+++ b/flink-python/pom.xml
@@ -34,7 +34,7 @@ under the License.
<packaging>jar</packaging>
<properties>
- <arrow.version>0.16.0</arrow.version>
+ <arrow.version>5.0.0</arrow.version>
</properties>
<dependencies>
@@ -88,6 +88,10 @@ under the License.
<groupId>org.apache.beam</groupId>
<artifactId>beam-vendor-bytebuddy-1_10_8</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.errorprone</groupId>
+
<artifactId>error_prone_annotations</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -100,7 +104,7 @@ under the License.
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>pemja</artifactId>
- <version>0.1.4</version>
+ <version>0.1.5</version>
</dependency>
<!-- Protobuf dependencies -->
@@ -141,6 +145,11 @@ under the License.
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-netty</artifactId>
+ <version>${arrow.version}</version>
+ </dependency>
<!-- test dependencies -->
@@ -368,11 +377,11 @@ under the License.
</excludes>
</filter>
<filter>
-
<artifact>org.apache.beam:beam-vendor-grpc-1_26_0</artifact>
+
<artifact>org.apache.beam:beam-vendor-grpc-1_43_2</artifact>
<excludes>
-
<exclude>org/apache/beam/vendor/grpc/v1p26p0/org/jboss/**</exclude>
+
<exclude>org/apache/beam/vendor/grpc/v1p43p2/org/jboss/**</exclude>
<exclude>schema/**</exclude>
-
<exclude>org/apache/beam/vendor/grpc/v1p26p0/org/eclipse/jetty/**</exclude>
+
<exclude>org/apache/beam/vendor/grpc/v1p43p2/org/eclipse/jetty/**</exclude>
</excludes>
</filter>
<filter>
diff --git a/flink-python/pyflink/fn_execution/beam/beam_operations.py
b/flink-python/pyflink/fn_execution/beam/beam_operations.py
index 3777c265638..601c0f51082 100644
--- a/flink-python/pyflink/fn_execution/beam/beam_operations.py
+++ b/flink-python/pyflink/fn_execution/beam/beam_operations.py
@@ -17,6 +17,7 @@
################################################################################
from apache_beam.portability import common_urns
from apache_beam.portability.api import beam_runner_api_pb2
+from apache_beam.runners import common
from apache_beam.runners.worker import bundle_processor, operation_specs
from apache_beam.utils import proto_utils
@@ -150,6 +151,7 @@ def _create_user_defined_function_operation(factory,
transform_proto, consumers,
input=None,
side_inputs=None,
output_coders=[output_coders[tag] for tag in output_tags])
+ name = common.NameContext(transform_proto.unique_name)
serialized_fn = spec.serialized_fn
if hasattr(serialized_fn, "key_type"):
@@ -172,7 +174,7 @@ def _create_user_defined_function_operation(factory,
transform_proto, consumers,
serialized_fn.map_state_write_cache_size)
return beam_operation_cls(
- transform_proto.unique_name,
+ name,
spec,
factory.counter_factory,
factory.state_sampler,
@@ -189,7 +191,7 @@ def _create_user_defined_function_operation(factory,
transform_proto, consumers,
serialized_fn.map_state_read_cache_size,
serialized_fn.map_state_write_cache_size)
return beam_operation_cls(
- transform_proto.unique_name,
+ name,
spec,
factory.counter_factory,
factory.state_sampler,
@@ -198,7 +200,7 @@ def _create_user_defined_function_operation(factory,
transform_proto, consumers,
keyed_state_backend)
else:
return beam_operation_cls(
- transform_proto.unique_name,
+ name,
spec,
factory.counter_factory,
factory.state_sampler,
diff --git a/flink-python/pyflink/table/tests/test_dependency.py
b/flink-python/pyflink/table/tests/test_dependency.py
index 121fe873380..ca91e0043e5 100644
--- a/flink-python/pyflink/table/tests/test_dependency.py
+++ b/flink-python/pyflink/table/tests/test_dependency.py
@@ -119,7 +119,7 @@ class StreamDependencyTests(DependencyTests,
PyFlinkStreamTableTestCase):
def test_set_requirements_without_cached_directory(self):
requirements_txt_path = os.path.join(self.tempdir, str(uuid.uuid4()))
with open(requirements_txt_path, 'w') as f:
- f.write("cloudpickle==1.2.2")
+ f.write("cloudpickle==2.0.0")
self.st_env.set_python_requirements(requirements_txt_path)
def check_requirements(i):
diff --git a/flink-python/setup.py b/flink-python/setup.py
index 5fdbb8652e6..9e8420cef1e 100644
--- a/flink-python/setup.py
+++ b/flink-python/setup.py
@@ -310,12 +310,12 @@ try:
author='Apache Software Foundation',
author_email='d...@flink.apache.org',
python_requires='>=3.6',
- install_requires=['py4j==0.10.9.3', 'python-dateutil==2.8.0',
'apache-beam==2.27.0',
- 'cloudpickle==1.2.2',
'avro-python3>=1.8.1,!=1.9.2,<1.10.0',
- 'pandas>=1.0,<1.2.0', 'pyarrow>=0.15.1,<3.0.0',
- 'pytz>=2018.3', 'numpy>=1.14.3,<1.20',
'fastavro>=0.21.4,<0.24',
+ install_requires=['py4j==0.10.9.3', 'python-dateutil==2.8.0',
'apache-beam==2.38.0',
+ 'cloudpickle==2.0.0',
'avro-python3>=1.8.1,!=1.9.2,<1.10.0',
+ 'pandas>=1.3.0', 'pyarrow>=5.0.0',
+ 'pytz>=2018.3', 'numpy>=1.21.4',
'fastavro>=0.21.4,<0.24',
'requests>=2.26.0', 'protobuf<3.18',
- 'pemja==0.1.4;python_full_version >= "3.7"',
+ 'pemja==0.1.5;python_full_version >= "3.7"',
'httplib2>=0.8,<0.19.0',
apache_flink_libraries_dependency],
cmdclass={'build_ext': build_ext},
tests_require=['pytest==4.4.1'],
diff --git
a/flink-python/src/main/java/org/apache/beam/runners/fnexecution/state/GrpcStateService.java
b/flink-python/src/main/java/org/apache/beam/runners/fnexecution/state/GrpcStateService.java
index a75a1c2d10b..c2270d83be4 100644
---
a/flink-python/src/main/java/org/apache/beam/runners/fnexecution/state/GrpcStateService.java
+++
b/flink-python/src/main/java/org/apache/beam/runners/fnexecution/state/GrpcStateService.java
@@ -20,9 +20,9 @@ package org.apache.beam.runners.fnexecution.state;
import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse;
import org.apache.beam.model.fnexecution.v1.BeamFnStateGrpc;
-import org.apache.beam.runners.fnexecution.FnService;
-import
org.apache.beam.vendor.grpc.v1p26p0.io.grpc.stub.ServerCallStreamObserver;
-import org.apache.beam.vendor.grpc.v1p26p0.io.grpc.stub.StreamObserver;
+import org.apache.beam.sdk.fn.server.FnService;
+import
org.apache.beam.vendor.grpc.v1p43p2.io.grpc.stub.ServerCallStreamObserver;
+import org.apache.beam.vendor.grpc.v1p43p2.io.grpc.stub.StreamObserver;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
diff --git
a/flink-python/src/main/java/org/apache/beam/vendor/grpc/v1p26p0/io/grpc/internal/SharedResourceHolder.java
b/flink-python/src/main/java/org/apache/beam/vendor/grpc/v1p43p2/io/grpc/internal/SharedResourceHolder.java
similarity index 98%
rename from
flink-python/src/main/java/org/apache/beam/vendor/grpc/v1p26p0/io/grpc/internal/SharedResourceHolder.java
rename to
flink-python/src/main/java/org/apache/beam/vendor/grpc/v1p43p2/io/grpc/internal/SharedResourceHolder.java
index 21afb6b71cb..c187bda4e2f 100644
---
a/flink-python/src/main/java/org/apache/beam/vendor/grpc/v1p26p0/io/grpc/internal/SharedResourceHolder.java
+++
b/flink-python/src/main/java/org/apache/beam/vendor/grpc/v1p43p2/io/grpc/internal/SharedResourceHolder.java
@@ -14,9 +14,9 @@
* limitations under the License.
*/
-package org.apache.beam.vendor.grpc.v1p26p0.io.grpc.internal;
+package org.apache.beam.vendor.grpc.v1p43p2.io.grpc.internal;
-import
org.apache.beam.vendor.grpc.v1p26p0.com.google.common.base.Preconditions;
+import
org.apache.beam.vendor.grpc.v1p43p2.com.google.common.base.Preconditions;
import javax.annotation.concurrent.ThreadSafe;
diff --git
a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamDataStreamPythonFunctionRunner.java
b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamDataStreamPythonFunctionRunner.java
index a2cc6a6e251..27edc588a46 100644
---
a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamDataStreamPythonFunctionRunner.java
+++
b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamDataStreamPythonFunctionRunner.java
@@ -179,7 +179,7 @@ public class BeamDataStreamPythonFunctionRunner extends
BeamPythonFunctionRunner
RunnerApi.FunctionSpec.newBuilder()
.setUrn(STATELESS_FUNCTION_URN)
.setPayload(
-
org.apache.beam.vendor.grpc.v1p26p0.com.google
+
org.apache.beam.vendor.grpc.v1p43p2.com.google
.protobuf.ByteString.copyFrom(
proto.toByteArray()))
.build());
@@ -196,7 +196,7 @@ public class BeamDataStreamPythonFunctionRunner extends
BeamPythonFunctionRunner
RunnerApi.FunctionSpec.newBuilder()
.setUrn(urn)
.setPayload(
-
org.apache.beam.vendor.grpc.v1p26p0.com.google
+
org.apache.beam.vendor.grpc.v1p43p2.com.google
.protobuf.ByteString.copyFrom(
proto.toByteArray()))
.build());
diff --git
a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java
b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java
index 9f7e558f2e9..650ef35295d 100644
---
a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java
+++
b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/BeamPythonFunctionRunner.java
@@ -71,7 +71,7 @@ import org.apache.beam.sdk.options.PortablePipelineOptions;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
import org.apache.beam.sdk.util.WindowedValue;
-import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct;
+import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct;
import
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -542,7 +542,7 @@ public abstract class BeamPythonFunctionRunner implements
PythonFunctionRunner {
RunnerApi.ExecutableStagePayload.WireCoderSetting.newBuilder()
.setUrn(getUrn(RunnerApi.StandardCoders.Enum.PARAM_WINDOWED_VALUE))
.setPayload(
-
org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString
+
org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString
.copyFrom(baos.toByteArray()))
.setInputOrOutputId(INPUT_COLLECTION_ID)
.build());
@@ -550,7 +550,7 @@ public abstract class BeamPythonFunctionRunner implements
PythonFunctionRunner {
RunnerApi.ExecutableStagePayload.WireCoderSetting.newBuilder()
.setUrn(getUrn(RunnerApi.StandardCoders.Enum.PARAM_WINDOWED_VALUE))
.setPayload(
-
org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString
+
org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString
.copyFrom(baos.toByteArray()))
.setInputOrOutputId(OUTPUT_COLLECTION_ID)
.build());
@@ -560,7 +560,7 @@ public abstract class BeamPythonFunctionRunner implements
PythonFunctionRunner {
RunnerApi.ExecutableStagePayload.WireCoderSetting.newBuilder()
.setUrn(getUrn(RunnerApi.StandardCoders.Enum.PARAM_WINDOWED_VALUE))
.setPayload(
-
org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf
+
org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf
.ByteString.copyFrom(baos.toByteArray()))
.setInputOrOutputId(entry.getKey())
.build());
diff --git
a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamBagStateHandler.java
b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamBagStateHandler.java
index 08051a544a3..0882b052605 100644
---
a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamBagStateHandler.java
+++
b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamBagStateHandler.java
@@ -26,7 +26,7 @@ import
org.apache.flink.runtime.state.internal.InternalListState;
import org.apache.flink.util.Preconditions;
import org.apache.beam.model.fnexecution.v1.BeamFnApi;
-import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;
import javax.annotation.Nullable;
diff --git
a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamMapStateHandler.java
b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamMapStateHandler.java
index 11e4810d094..c813c2a4153 100644
---
a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamMapStateHandler.java
+++
b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamMapStateHandler.java
@@ -28,7 +28,7 @@ import org.apache.flink.python.PythonOptions;
import org.apache.flink.streaming.api.utils.ByteArrayWrapper;
import org.apache.beam.model.fnexecution.v1.BeamFnApi;
-import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;
import java.util.HashMap;
import java.util.Iterator;
diff --git
a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamStateRequestHandler.java
b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamStateRequestHandler.java
index cbb56965515..43fbf95b731 100644
---
a/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamStateRequestHandler.java
+++
b/flink-python/src/main/java/org/apache/flink/streaming/api/runners/python/beam/state/BeamStateRequestHandler.java
@@ -28,8 +28,8 @@ import org.apache.flink.streaming.api.utils.ByteArrayWrapper;
import org.apache.beam.model.fnexecution.v1.BeamFnApi;
import org.apache.beam.runners.fnexecution.state.StateRequestHandler;
-import org.apache.beam.vendor.grpc.v1p26p0.com.google.common.base.Charsets;
-import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.grpc.v1p43p2.com.google.common.base.Charsets;
+import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString;
import javax.annotation.Nullable;
diff --git
a/flink-python/src/main/java/org/apache/flink/streaming/api/utils/ProtoUtils.java
b/flink-python/src/main/java/org/apache/flink/streaming/api/utils/ProtoUtils.java
index 0f63ebb8f2a..9921174a83b 100644
---
a/flink-python/src/main/java/org/apache/flink/streaming/api/utils/ProtoUtils.java
+++
b/flink-python/src/main/java/org/apache/flink/streaming/api/utils/ProtoUtils.java
@@ -284,7 +284,7 @@ public enum ProtoUtils {
RunnerApi.FunctionSpec.newBuilder()
.setUrn(FLINK_CODER_URN)
.setPayload(
-
org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf
+
org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf
.ByteString.copyFrom(
coderInfoDescriptor.toByteArray()))
.build())
diff --git
a/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/beam/BeamTablePythonFunctionRunner.java
b/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/beam/BeamTablePythonFunctionRunner.java
index 97ccf8917a7..888c3c4f9be 100644
---
a/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/beam/BeamTablePythonFunctionRunner.java
+++
b/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/beam/BeamTablePythonFunctionRunner.java
@@ -92,7 +92,7 @@ public class BeamTablePythonFunctionRunner extends
BeamPythonFunctionRunner {
RunnerApi.FunctionSpec.newBuilder()
.setUrn(functionUrn)
.setPayload(
-
org.apache.beam.vendor.grpc.v1p26p0.com.google
+
org.apache.beam.vendor.grpc.v1p43p2.com.google
.protobuf.ByteString.copyFrom(
userDefinedFunctionProto.toByteArray()))
.build())
diff --git a/flink-python/src/main/resources/META-INF/NOTICE
b/flink-python/src/main/resources/META-INF/NOTICE
index 182971f3520..2d636b070a3 100644
--- a/flink-python/src/main/resources/META-INF/NOTICE
+++ b/flink-python/src/main/resources/META-INF/NOTICE
@@ -9,25 +9,26 @@ This project bundles the following dependencies under the
Apache Software Licens
- com.fasterxml.jackson.core:jackson-annotations:2.13.2
- com.fasterxml.jackson.core:jackson-core:2.13.2
- com.fasterxml.jackson.core:jackson-databind:2.13.2.2
-- com.google.flatbuffers:flatbuffers-java:1.9.0
-- io.netty:netty-buffer:4.1.70.Final
-- io.netty:netty-common:4.1.70.Final
+- com.google.flatbuffers:flatbuffers-java:1.12.0
- joda-time:joda-time:2.5
-- org.apache.arrow:arrow-format:0.16.0
-- org.apache.arrow:arrow-memory:0.16.0
-- org.apache.arrow:arrow-vector:0.16.0
-- org.apache.beam:beam-model-fn-execution:2.27.0
-- org.apache.beam:beam-model-job-management:2.27.0
-- org.apache.beam:beam-model-pipeline:2.27.0
-- org.apache.beam:beam-runners-core-construction-java:2.27.0
-- org.apache.beam:beam-runners-core-java:2.27.0
-- org.apache.beam:beam-runners-java-fn-execution:2.27.0
-- org.apache.beam:beam-sdks-java-core:2.27.0
-- org.apache.beam:beam-sdks-java-fn-execution:2.27.0
-- org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.27.0
+- org.apache.arrow:arrow-format:5.0.0
+- org.apache.arrow:arrow-memory:5.0.0
+- org.apache.arrow:arrow-memory-core:5.0.0
+- org.apache.arrow:arrow-memory-netty:5.0.0
+- org.apache.arrow:arrow-vector:5.0.0
+- org.apache.beam:beam-model-fn-execution:2.38.0
+- org.apache.beam:beam-model-job-management:2.38.0
+- org.apache.beam:beam-model-pipeline:2.38.0
+- org.apache.beam:beam-runners-core-construction-java:2.38.0
+- org.apache.beam:beam-runners-core-java:2.38.0
+- org.apache.beam:beam-runners-java-fn-execution:2.38.0
+- org.apache.beam:beam-sdks-java-core:2.38.0
+- org.apache.beam:beam-sdks-java-fn-execution:2.38.0
+- org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+- org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.38.0
- org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-- org.apache.beam:beam-vendor-grpc-1_26_0:0.3
-- com.alibaba:pemja:0.1.4
+- org.apache.beam:beam-vendor-grpc-1_43_2:0.1
+- com.alibaba:pemja:0.1.5
This project bundles the following dependencies under the BSD license.
See bundled license files for details
@@ -47,26 +48,26 @@ The bundled Apache Beam dependencies bundle the following
dependencies under the
- com.google.code.gson:gson:2.8.6
- com.google.guava:guava:26.0-jre
- com.ning:compress-lzf:1.0.3
-- io.grpc:grpc-auth:1.26.0
-- io.grpc:grpc-core:1.26.0
-- io.grpc:grpc-context:1.26.0
-- io.grpc:grpc-netty:1.26.0
-- io.grpc:grpc-protobuf:1.26.0
-- io.grpc:grpc-stub:1.26.0
-- io.grpc:grpc-testing:1.26.0
-- io.netty:netty-buffer:4.1.51.Final
-- io.netty:netty-codec:4.1.51.Final
-- io.netty:netty-codec-http:4.1.51.Final
-- io.netty:netty-codec-http2:4.1.51.Final
-- io.netty:netty-codec-socks:4.1.51.Final
-- io.netty:netty-common:4.1.51.Final
-- io.netty:netty-handler:4.1.51.Final
-- io.netty:netty-handler-proxy:4.1.51.Final
-- io.netty:netty-resolver:4.1.51.Final
-- io.netty:netty-transport:4.1.51.Final
-- io.netty:netty-transport-native-epoll:4.1.51.Final
-- io.netty:netty-transport-native-unix-common:4.1.51.Final
-- io.netty:netty-tcnative-boringssl-static:2.0.33.Final
+- io.grpc:grpc-auth:1.43.2
+- io.grpc:grpc-core:1.43.2
+- io.grpc:grpc-context:1.43.2
+- io.grpc:grpc-netty:1.43.2
+- io.grpc:grpc-protobuf:1.43.2
+- io.grpc:grpc-stub:1.43.2
+- io.grpc:grpc-testing:1.43.2
+- io.netty:netty-buffer:4.1.70.Final
+- io.netty:netty-codec:4.1.70.Final
+- io.netty:netty-codec-http:4.1.70.Final
+- io.netty:netty-codec-http2:4.1.70.Final
+- io.netty:netty-codec-socks:4.1.70.Final
+- io.netty:netty-common:4.1.70.Final
+- io.netty:netty-handler:4.1.70.Final
+- io.netty:netty-handler-proxy:4.1.70.Final
+- io.netty:netty-resolver:4.1.70.Final
+- io.netty:netty-transport:4.1.70.Final
+- io.netty:netty-transport-native-epoll:4.1.70.Final
+- io.netty:netty-transport-native-unix-common:4.1.70.Final
+- io.netty:netty-tcnative-boringssl-static:2.0.44.Final
- io.opencensus:opencensus-api:0.24.0
- io.opencensus:opencensus-contrib-grpc-metrics:0.24.0
- io.perfmark:perfmark-api:0.19.0
diff --git
a/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonAggregateFunctionRunner.java
b/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonAggregateFunctionRunner.java
index a1667c9c9c3..4367892f627 100644
---
a/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonAggregateFunctionRunner.java
+++
b/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonAggregateFunctionRunner.java
@@ -33,7 +33,7 @@ import
org.apache.flink.table.runtime.runners.python.beam.BeamTablePythonFunctio
import org.apache.flink.table.types.logical.RowType;
import org.apache.beam.runners.fnexecution.control.JobBundleFactory;
-import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct;
+import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct;
import java.util.ArrayList;
import java.util.LinkedList;
diff --git
a/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonScalarFunctionRunner.java
b/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonScalarFunctionRunner.java
index f1f6bb6710d..617cf45172f 100644
---
a/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonScalarFunctionRunner.java
+++
b/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonScalarFunctionRunner.java
@@ -26,7 +26,7 @@ import
org.apache.flink.table.runtime.runners.python.beam.BeamTablePythonFunctio
import org.apache.flink.table.types.logical.RowType;
import org.apache.beam.runners.fnexecution.control.JobBundleFactory;
-import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct;
+import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct;
import java.util.LinkedList;
import java.util.List;
diff --git
a/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonTableFunctionRunner.java
b/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonTableFunctionRunner.java
index 53713d8b72b..1a9d30dbaa8 100644
---
a/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonTableFunctionRunner.java
+++
b/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughPythonTableFunctionRunner.java
@@ -26,7 +26,7 @@ import
org.apache.flink.table.runtime.runners.python.beam.BeamTablePythonFunctio
import org.apache.flink.table.types.logical.RowType;
import org.apache.beam.runners.fnexecution.control.JobBundleFactory;
-import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct;
+import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct;
import java.util.LinkedList;
import java.util.List;
diff --git
a/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughStreamAggregatePythonFunctionRunner.java
b/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughStreamAggregatePythonFunctionRunner.java
index 9698264a58f..0c2b21051dd 100644
---
a/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughStreamAggregatePythonFunctionRunner.java
+++
b/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughStreamAggregatePythonFunctionRunner.java
@@ -28,7 +28,7 @@ import
org.apache.flink.table.runtime.runners.python.beam.BeamTablePythonFunctio
import org.apache.flink.table.types.logical.RowType;
import org.apache.beam.runners.fnexecution.control.JobBundleFactory;
-import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct;
+import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct;
import java.util.LinkedList;
import java.util.List;
diff --git
a/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughStreamGroupWindowAggregatePythonFunctionRunner.java
b/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughStreamGroupWindowAggregatePythonFunctionRunner.java
index a76fc1c7fd8..64eb8657a27 100644
---
a/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughStreamGroupWindowAggregatePythonFunctionRunner.java
+++
b/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughStreamGroupWindowAggregatePythonFunctionRunner.java
@@ -28,7 +28,7 @@ import
org.apache.flink.table.runtime.runners.python.beam.BeamTablePythonFunctio
import org.apache.flink.table.types.logical.RowType;
import org.apache.beam.runners.fnexecution.control.JobBundleFactory;
-import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct;
+import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct;
import static
org.apache.flink.streaming.api.utils.ProtoUtils.createFlattenRowTypeCoderInfoDescriptorProto;
diff --git
a/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughStreamTableAggregatePythonFunctionRunner.java
b/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughStreamTableAggregatePythonFunctionRunner.java
index 77ea13237c5..4a962662f09 100644
---
a/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughStreamTableAggregatePythonFunctionRunner.java
+++
b/flink-python/src/test/java/org/apache/flink/table/runtime/utils/PassThroughStreamTableAggregatePythonFunctionRunner.java
@@ -28,7 +28,7 @@ import
org.apache.flink.table.runtime.runners.python.beam.BeamTablePythonFunctio
import org.apache.flink.table.types.logical.RowType;
import org.apache.beam.runners.fnexecution.control.JobBundleFactory;
-import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct;
+import org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct;
import java.util.Arrays;
import java.util.LinkedList;
diff --git a/pom.xml b/pom.xml
index b7500689754..a2646ee8038 100644
--- a/pom.xml
+++ b/pom.xml
@@ -135,7 +135,7 @@ under the License.
<hamcrest.version>1.3</hamcrest.version>
<assertj.version>3.21.0</assertj.version>
<py4j.version>0.10.9.3</py4j.version>
- <beam.version>2.27.0</beam.version>
+ <beam.version>2.38.0</beam.version>
<protoc.version>3.17.3</protoc.version>
<okhttp.version>3.14.9</okhttp.version>
<testcontainers.version>1.17.2</testcontainers.version>
diff --git a/tools/releasing/NOTICE-binary_PREAMBLE.txt
b/tools/releasing/NOTICE-binary_PREAMBLE.txt
index 6228b0bdb9d..feee546e601 100644
--- a/tools/releasing/NOTICE-binary_PREAMBLE.txt
+++ b/tools/releasing/NOTICE-binary_PREAMBLE.txt
@@ -16,4 +16,4 @@ This project bundles the following dependencies under the
Apache Software Licens
This project bundles the following dependencies under the BSD license.
See bundled license files for details.
-- cloudpickle:1.2.2
+- cloudpickle:2.0.0
