[ https://issues.apache.org/jira/browse/FLINK-14944?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16982108#comment-16982108 ]
sunjincheng commented on FLINK-14944: ------------------------------------- Hi [~dian.fu] Thanks for report and trace this issue, in the log we can find that " ConnectionResetError: [Errno 104] Connection reset by peer", So, It seems that this has something to do with the network(some times, not every time). Is it possible to add a check for network exception to the test and print the log, then ensure that the test passes then does not affect existing PRs tests? And I am agree that we should find out the real reason for this issue, and fix it using the final solution. What do you think? > Unstable test FlinkFnExecutionSyncTests.test_flink_fn_execution_pb2_synced > -------------------------------------------------------------------------- > > Key: FLINK-14944 > URL: https://issues.apache.org/jira/browse/FLINK-14944 > Project: Flink > Issue Type: Bug > Components: API / Python > Affects Versions: 1.10.0 > Reporter: Dian Fu > Priority: Major > Fix For: 1.10.0 > > > This tests failed occasionally: > {code:java} > force = 'True', output_dir = '/tmp/tmpkh6rmeig' > def generate_proto_files(force=True, > output_dir=DEFAULT_PYTHON_OUTPUT_PATH): > try: > import grpc_tools # noqa # pylint: disable=unused-import > except ImportError: > warnings.warn('Installing grpcio-tools is recommended for > development.') > > proto_dirs = [os.path.join(PYFLINK_ROOT_PATH, path) for path in > PROTO_PATHS] > proto_files = sum( > [glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], []) > out_dir = os.path.join(PYFLINK_ROOT_PATH, output_dir) > out_files = [path for path in glob.glob(os.path.join(out_dir, > '*_pb2.py'))] > > if out_files and not proto_files and not force: > # We have out_files but no protos; assume they're up to date. > # This is actually the common case (e.g. installation from an > sdist). > logging.info('No proto files; using existing generated files.') > return > > elif not out_files and not proto_files: > raise RuntimeError( > 'No proto files found in %s.' % proto_dirs) > > # Regenerate iff the proto files or this file are newer. > elif force or not out_files or len(out_files) < len(proto_files) or ( > min(os.path.getmtime(path) for path in out_files) > <= max(os.path.getmtime(path) > for path in proto_files + > [os.path.realpath(__file__)])): > try: > > from grpc_tools import protoc > E ModuleNotFoundError: No module named 'grpc_tools' > pyflink/gen_protos.py:70: ModuleNotFoundError > During handling of the above exception, another exception occurred: > self = > <pyflink.fn_execution.tests.test_flink_fn_execution_pb2_synced.FlinkFnExecutionSyncTests > testMethod=test_flink_fn_execution_pb2_synced> > def test_flink_fn_execution_pb2_synced(self): > > generate_proto_files('True', self.tempdir) > pyflink/fn_execution/tests/test_flink_fn_execution_pb2_synced.py:35: > _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ > _ > pyflink/gen_protos.py:83: in generate_proto_files > target=_install_grpcio_tools_and_generate_proto_files(force, output_dir)) > pyflink/gen_protos.py:131: in _install_grpcio_tools_and_generate_proto_files > '--upgrade', GRPC_TOOLS, "-I"]) > _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ > _ > popenargs = > (['/home/travis/build/apache/flink/flink-python/.tox/py36/bin/python', '-m', > 'pip', 'install', '--prefix', > '/home/travis/build/apache/flink/flink-python/pyflink/../.eggs/grpcio-wheels', > ...],) > kwargs = {}, retcode = 2 > cmd = ['/home/travis/build/apache/flink/flink-python/.tox/py36/bin/python', > '-m', 'pip', 'install', '--prefix', > '/home/travis/build/apache/flink/flink-python/pyflink/../.eggs/grpcio-wheels', > ...] > def check_call(*popenargs, **kwargs): > """Run command with arguments. Wait for command to complete. If > the exit code was zero then return, otherwise raise > CalledProcessError. The CalledProcessError object will have the > return code in the returncode attribute. > > The arguments are the same as for the call function. Example: > > check_call(["ls", "-l"]) > """ > retcode = call(*popenargs, **kwargs) > if retcode: > cmd = kwargs.get("args") > if cmd is None: > cmd = popenargs[0] > > raise CalledProcessError(retcode, cmd) > E subprocess.CalledProcessError: Command > '['/home/travis/build/apache/flink/flink-python/.tox/py36/bin/python', '-m', > 'pip', 'install', '--prefix', > '/home/travis/build/apache/flink/flink-python/pyflink/../.eggs/grpcio-wheels', > '--build', > '/home/travis/build/apache/flink/flink-python/pyflink/../.eggs/grpcio-wheels-build', > '--upgrade', 'grpcio-tools>=1.3.5,<=1.14.2', '-I']' returned non-zero exit > status 2. > dev/.conda/envs/3.6/lib/python3.6/subprocess.py:311: CalledProcessError > ---------------------------- Captured stdout setup > ----------------------------- > 2019-11-25 14:25:21,766 - root - INFO - Using > /home/travis/build/apache/flink/build-target as FLINK_HOME... > ------------------------------ Captured log setup > ------------------------------ > INFO root:test_case_utils.py:75 Using > /home/travis/build/apache/flink/build-target as FLINK_HOME... > ----------------------------- Captured stdout call > ----------------------------- > 2019-11-25 14:25:21,768 - root - WARNING - Installing grpcio-tools into > /home/travis/build/apache/flink/flink-python/pyflink/../.eggs/grpcio-wheels > Collecting grpcio-tools<=1.14.2,>=1.3.5 > Downloading > https://files.pythonhosted.org/packages/df/a6/ec4baac1ab265857800cfdbd07a05db5fa5e159ee44fd41b358784527d94/grpcio_tools-1.14.2-cp36-cp36m-manylinux1_x86_64.whl > (22.2MB) > ----------------------------- Captured stderr call > ----------------------------- > Exception: > Traceback (most recent call last): > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_vendor/urllib3/response.py", > line 302, in _error_catcher > yield > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_vendor/urllib3/response.py", > line 384, in read > data = self._fp.read(amt) > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_vendor/cachecontrol/filewrapper.py", > line 60, in read > data = self.__fp.read(amt) > File > "/home/travis/build/apache/flink/flink-python/dev/.conda/envs/3.6/lib/python3.6/http/client.py", > line 459, in read > n = self.readinto(b) > File > "/home/travis/build/apache/flink/flink-python/dev/.conda/envs/3.6/lib/python3.6/http/client.py", > line 503, in readinto > n = self.fp.readinto(b) > File > "/home/travis/build/apache/flink/flink-python/dev/.conda/envs/3.6/lib/python3.6/socket.py", > line 586, in readinto > return self._sock.recv_into(b) > File > "/home/travis/build/apache/flink/flink-python/dev/.conda/envs/3.6/lib/python3.6/ssl.py", > line 1012, in recv_into > return self.read(nbytes, buffer) > File > "/home/travis/build/apache/flink/flink-python/dev/.conda/envs/3.6/lib/python3.6/ssl.py", > line 874, in read > return self._sslobj.read(len, buffer) > File > "/home/travis/build/apache/flink/flink-python/dev/.conda/envs/3.6/lib/python3.6/ssl.py", > line 631, in read > v = self._sslobj.read(len, buffer) > ConnectionResetError: [Errno 104] Connection reset by peer > During handling of the above exception, another exception occurred: > Traceback (most recent call last): > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/basecommand.py", > line 228, in main > status = self.run(options, args) > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/commands/install.py", > line 291, in run > resolver.resolve(requirement_set) > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/resolve.py", > line 103, in resolve > self._resolve_one(requirement_set, req) > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/resolve.py", > line 257, in _resolve_one > abstract_dist = self._get_abstract_dist_for(req_to_install) > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/resolve.py", > line 210, in _get_abstract_dist_for > self.require_hashes > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/operations/prepare.py", > line 310, in prepare_linked_requirement > progress_bar=self.progress_bar > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/download.py", > line 837, in unpack_url > progress_bar=progress_bar > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/download.py", > line 674, in unpack_http_url > progress_bar) > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/download.py", > line 898, in _download_http_url > _download_url(resp, link, content_file, hashes, progress_bar) > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/download.py", > line 618, in _download_url > hashes.check_against_chunks(downloaded_chunks) > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/utils/hashes.py", > line 48, in check_against_chunks > for chunk in chunks: > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/download.py", > line 586, in written_chunks > for chunk in chunks: > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/utils/ui.py", > line 159, in iter > for x in it: > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_internal/download.py", > line 575, in resp_read > decode_content=False): > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_vendor/urllib3/response.py", > line 436, in stream > data = self.read(amt=amt, decode_content=decode_content) > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_vendor/urllib3/response.py", > line 401, in read > raise IncompleteRead(self._fp_bytes_read, self.length_remaining) > File > "/home/travis/build/apache/flink/flink-python/dev/.conda/envs/3.6/lib/python3.6/contextlib.py", > line 99, in __exit__ > self.gen.throw(type, value, traceback) > File > "/home/travis/build/apache/flink/flink-python/.tox/py36/lib/python3.6/site-packages/pip/_vendor/urllib3/response.py", > line 320, in _error_catcher > raise ProtocolError('Connection broken: %r' % e, e) > pip._vendor.urllib3.exceptions.ProtocolError: ("Connection broken: > ConnectionResetError(104, 'Connection reset by peer')", > ConnectionResetError(104, 'Connection reset by peer')) > You are using pip version 10.0.1, however version 19.3.1 is available. > You should consider upgrading via the 'pip install --upgrade pip' command. > {code} > instance: [https://api.travis-ci.org/v3/job/616685590/log.txt] -- This message was sent by Atlassian Jira (v8.3.4#803005)