[ 
https://issues.apache.org/jira/browse/BEAM-7546?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16862312#comment-16862312
 ] 

Valentyn Tymofieiev commented on BEAM-7546:
-------------------------------------------

Hi [~angoenka], is this something you can take a look or triage? Thanks.
cc: [~mxm] [~robertwb] [~altay]

> Portable WordCount-on-Flink Precommit is flaky - temporary folder not found.
> ----------------------------------------------------------------------------
>
>                 Key: BEAM-7546
>                 URL: https://issues.apache.org/jira/browse/BEAM-7546
>             Project: Beam
>          Issue Type: Bug
>          Components: runner-flink
>            Reporter: Valentyn Tymofieiev
>            Assignee: Ankur Goenka
>            Priority: Major
>
> On a few occasions I see this test fail due to a temp directory being missing.
> Sample scan from 
> https://builds.apache.org/job/beam_PreCommit_Portable_Python_Cron/745/: 
> https://scans.gradle.com/s/3ra4xw4hqvlyw/console-log?task=:sdks:python:portableWordCountBatch
> {noformat}
> [grpc-default-executor-0] ERROR sdk_worker._execute - Error processing 
> instruction 8. Original traceback is
> Traceback (most recent call last):
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 157, in _execute
>     response = task()
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 190, in <lambda>
>     self._execute(lambda: worker.do_instruction(work), work)
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 342, in do_instruction
>     request.instruction_id)
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
>  line 368, in process_bundle
>     bundle_processor.process_bundle(instruction_id))
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
>  line 589, in process_bundle
>     ].process_encoded(data.data)
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
>  line 143, in process_encoded
>     self.output(decoded_value)
>   File "apache_beam/runners/worker/operations.py", line 255, in 
> apache_beam.runners.worker.operations.Operation.output
>     def output(self, windowed_value, output_index=0):
>   File "apache_beam/runners/worker/operations.py", line 256, in 
> apache_beam.runners.worker.operations.Operation.output
>     cython.cast(Receiver, 
> self.receivers[output_index]).receive(windowed_value)
>   File "apache_beam/runners/worker/operations.py", line 143, in 
> apache_beam.runners.worker.operations.SingletonConsumerSet.receive
>     self.consumer.process(windowed_value)
>   File "apache_beam/runners/worker/operations.py", line 593, in 
> apache_beam.runners.worker.operations.DoOperation.process
>     with self.scoped_process_state:
>   File "apache_beam/runners/worker/operations.py", line 594, in 
> apache_beam.runners.worker.operations.DoOperation.process
>     delayed_application = self.dofn_receiver.receive(o)
>   File "apache_beam/runners/common.py", line 778, in 
> apache_beam.runners.common.DoFnRun
> ner.receive
>     self.process(windowed_value)
>   File "apache_beam/runners/common.py", line 784, in 
> apache_beam.runners.common.DoFnRunner.process
>     self._reraise_augmented(exn)
>   File "apache_beam/runners/common.py", line 851, in 
> apache_beam.runners.common.DoFnRunner._reraise_augmented
>     raise_with_traceback(new_exn)
>   File "apache_beam/runners/common.py", line 782, in 
> apache_beam.runners.common.DoFnRunner.process
>     return self.do_fn_invoker.invoke_process(windowed_value)
>   File "apache_beam/runners/common.py", line 594, in 
> apache_beam.runners.common.PerWindowInvoker.invoke_process
>     self._invoke_process_per_window(
>   File "apache_beam/runners/common.py", line 666, in 
> apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
>     windowed_value, self.process_method(*args_for_process))
>   File "/usr/local/lib/python2.7/site-packages/apache_beam/io/iobase.py", 
> line 1041, in process
>     self.writer = self.sink.open_writer(init_result, str(uuid.uuid4()))
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/options/value_provider.py",
>  line 137, in _f
>     return fnc(self, *args, **kwargs)
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/io/filebasedsink.py", 
> line 185, in open_writer
>     return FileBasedSinkWriter(self, os.path.join(init_result, uid) + suffix)
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/io/filebasedsink.py", 
> line 389, in __init__
>     self.temp_handle = self.sink.open(temp_shard_path)
>   File "/usr/local/lib/python2.7/site-packages/apache_beam/io/textio.py", 
> line 391, in open
>     file_handle = super(_TextSink, self).open(temp_path)
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/options/value_provider.py",
>  line 137, in _f
>     return fnc(self, *args, **kwargs)
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/io/filebasedsink.py", 
> line 129, in open
>     return FileSystems.create(temp_path, self.mime_type, 
> self.compression_type)
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/io/filesystems.py", line 
> 203, in create
>     return file
> system.create(path, mime_type, compression_type)
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/io/localfilesystem.py", 
> line 151, in create
>     return self._path_open(path, 'wb', mime_type, compression_type)
>   File 
> "/usr/local/lib/python2.7/site-packages/apache_beam/io/localfilesystem.py", 
> line 134, in _path_open
>     raw_file = open(path, mode)
> RuntimeError: IOError: [Errno 2] No such file or directory: 
> '/tmp/beam-temp-py-wordcount-direct-d3ca98a287bd11e997e842010a80006f/213ae725-add1-4ded-8e9b-ee9ac1ab6fd0.py-wordcount-direct'
>  [while running 'write/Write/WriteImpl/WriteBundles']
> {noformat}
> There is also following exception earlier.
> {noformat}
> :sdks:python:portableWordCountBatch FAILED
> INFO:root:Using latest locally built Python SDK docker image.
> INFO:root:==================== <function lift_combiners at 0x7f63550fcf50> 
> ====================
> INFO:root:==================== <function expand_sdf at 0x7f63550f5050> 
> ====================
> [main] INFO org.apache.beam.runners.fnexecution.jobsubmission.JobServerDriver 
> - ArtifactStagingService started on localhost:34803
> [main] INFO org.apache.beam.runners.fnexecution.jobsubmission.JobServerDriver 
> - Java ExpansionService started on localhost:59151
> [main] INFO org.apache.beam.runners.fnexecution.jobsubmission.JobServerDriver 
> - JobService started on localhost:57157
> [grpc-default-executor-0] ERROR 
> org.apache.beam.runners.fnexecution.jobsubmission.InMemoryJobService - 
> Encountered Unexpected Exception for Invocation 
> job_8dd6b809-0f47-4943-9d3e-4587db962ab9
> org.apache.beam.vendor.grpc.v1p13p1.io.grpc.StatusException: NOT_FOUND
>       at 
> org.apache.beam.vendor.grpc.v1p13p1.io.grpc.Status.asException(Status.java:534)
>       at 
> org.apache.beam.runners.fnexecution.jobsubmission.InMemoryJobService.getInvocation(InMemoryJobService.java:341)
>       at 
> org.apache.beam.runners.fnexecution.jobsubmission.InMemoryJobService.getStateStream(InMemoryJobService.java:262)
>       at 
> org.apache.beam.model.jobmanagement.v1.JobServiceGrpc$MethodHandlers.invoke(JobServiceGrpc.java:770)
>       at 
> org.apache.beam.vendor.grpc.v1p13p1.io.grpc.stub.ServerCalls$UnaryServerCallHandler$UnaryServerCallListener.onHalfClose(ServerCalls.java:171)
>       at 
> org.apache.beam.vendor.grpc.v1p13p1.io.grpc.PartialForwardingServerCallListener.onHalfClose(PartialForwardingServerCallListener.java:35)
>       at 
> org.apache.beam.vendor.grpc.v1p13p1.io.grpc.ForwardingServerCallListener.onHalfClose(ForwardingServerCallListener.java:23)
>       at 
> org.apache.beam.vendor.grpc.v1p13p1.io.grpc.ForwardingServerCallListener$SimpleForwardingServerCallListener.onHalfClose(ForwardingServerCallListener.java:40)
>       at 
> org.apache.beam.vendor.grpc.v1p13p1.io.grpc.Contexts$ContextualizedServerCallListener.onHalfClose(Contexts.java:86)
>       at 
> org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.halfClosed(ServerCallImpl.java:283)
>       at 
> org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1HalfClosed.runInContext(ServerImpl.java:707)
>       at 
> org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
>       at 
> org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
>       at 
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
>       at 
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
>       at java.lang.Thread.run(Thread.java:748)
> {noformat}



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Reply via email to