Mark Liu created BEAM-1577: ------------------------------ Summary: Python BigqueryTornadoes Example Failed Key: BEAM-1577 URL: https://issues.apache.org/jira/browse/BEAM-1577 Project: Beam Issue Type: Bug Components: sdk-py Affects Versions: 0.6.0 Reporter: Mark Liu Assignee: Ahmet Altay
Running bigquery_tornadoes on DataflowRunner failed with following commands: python -m apache_beam.examples.cookbook.bigquery_tornadoes \ --project google.com:clouddfe \ --runner DataflowRunner \ --staging_location gs://.../tmp/python/staging \ --temp_location gs://.../tmp/python/temp \ --output BigQueryTornadoesIT.monthly_tornadoes_0001 \ --sdk_location dist/apache-beam-0.6.0.dev0.tar.gz JobName: beamapp-markliu-0301012437-463604 Job Link: https://pantheon.corp.google.com/dataflow/job/2017-02-28_17_24_39-6087587560567828169?project=google.com:clouddfe&organizationId=433637338589 Full stack trace: {code} Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py", line 544, in do_work work_executor.execute() File "dataflow_worker/executor.py", line 971, in dataflow_worker.executor.MapTaskExecutor.execute (dataflow_worker/executor.c:30533) with op.scoped_metrics_container: File "dataflow_worker/executor.py", line 972, in dataflow_worker.executor.MapTaskExecutor.execute (dataflow_worker/executor.c:30481) op.start() File "dataflow_worker/executor.py", line 207, in dataflow_worker.executor.ReadOperation.start (dataflow_worker/executor.c:8758) def start(self): File "dataflow_worker/executor.py", line 208, in dataflow_worker.executor.ReadOperation.start (dataflow_worker/executor.c:8663) with self.scoped_start_state: File "dataflow_worker/executor.py", line 213, in dataflow_worker.executor.ReadOperation.start (dataflow_worker/executor.c:8579) with self.spec.source.reader() as reader: File "dataflow_worker/executor.py", line 223, in dataflow_worker.executor.ReadOperation.start (dataflow_worker/executor.c:8524) self.output(windowed_value) File "dataflow_worker/executor.py", line 151, in dataflow_worker.executor.Operation.output (dataflow_worker/executor.c:6317) cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value) File "dataflow_worker/executor.py", line 84, in dataflow_worker.executor.ConsumerSet.receive (dataflow_worker/executor.c:4021) cython.cast(Operation, consumer).process(windowed_value) File "dataflow_worker/executor.py", line 544, in dataflow_worker.executor.DoOperation.process (dataflow_worker/executor.c:18474) with self.scoped_process_state: File "dataflow_worker/executor.py", line 545, in dataflow_worker.executor.DoOperation.process (dataflow_worker/executor.c:18428) self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 195, in apache_beam.runners.common.DoFnRunner.receive (apache_beam/runners/common.c:5142) self.process(windowed_value) File "apache_beam/runners/common.py", line 267, in apache_beam.runners.common.DoFnRunner.process (apache_beam/runners/common.c:7201) self.reraise_augmented(exn) File "apache_beam/runners/common.py", line 279, in apache_beam.runners.common.DoFnRunner.reraise_augmented (apache_beam/runners/common.c:7590) raise type(exn), args, sys.exc_info()[2] File "apache_beam/runners/common.py", line 263, in apache_beam.runners.common.DoFnRunner.process (apache_beam/runners/common.c:7090) self._dofn_simple_invoker(element) File "apache_beam/runners/common.py", line 198, in apache_beam.runners.common.DoFnRunner._dofn_simple_invoker (apache_beam/runners/common.c:5288) self._process_outputs(element, self.dofn_process(element.value)) File "apache_beam/runners/common.py", line 326, in apache_beam.runners.common.DoFnRunner._process_outputs (apache_beam/runners/common.c:8563) self.main_receivers.receive(windowed_value) File "dataflow_worker/executor.py", line 82, in dataflow_worker.executor.ConsumerSet.receive (dataflow_worker/executor.c:3987) self.update_counters_start(windowed_value) File "dataflow_worker/executor.py", line 88, in dataflow_worker.executor.ConsumerSet.update_counters_start (dataflow_worker/executor.c:4207) self.opcounter.update_from(windowed_value) File "dataflow_worker/opcounters.py", line 57, in dataflow_worker.opcounters.OperationCounters.update_from (dataflow_worker/opcounters.c:2396) self.do_sample(windowed_value) File "dataflow_worker/opcounters.py", line 75, in dataflow_worker.opcounters.OperationCounters.do_sample (dataflow_worker/opcounters.c:3017) self.coder_impl.get_estimated_size_and_observables(windowed_value)) File "apache_beam/coders/coder_impl.py", line 695, in apache_beam.coders.coder_impl.WindowedValueCoderImpl.get_estimated_size_and_observables (apache_beam/coders/coder_impl.c:22894) def get_estimated_size_and_observables(self, value, nested=False): File "apache_beam/coders/coder_impl.py", line 704, in apache_beam.coders.coder_impl.WindowedValueCoderImpl.get_estimated_size_and_observables (apache_beam/coders/coder_impl.c:22613) self._value_coder.get_estimated_size_and_observables( File "apache_beam/coders/coder_impl.py", line 494, in apache_beam.coders.coder_impl.AbstractComponentCoderImpl.get_estimated_size_and_observables (apache_beam/coders/coder_impl.c:18402) c.get_estimated_size_and_observables( File "apache_beam/coders/coder_impl.py", line 247, in apache_beam.coders.coder_impl.FastPrimitivesCoderImpl.get_estimated_size_and_observables (apache_beam/coders/coder_impl.c:9564) out = ByteCountingOutputStream() File "apache_beam/coders/stream.pyx", line 28, in apache_beam.coders.stream.OutputStream.__cinit__ (apache_beam/coders/stream.c:1241) self.buffer_size = 1024 AttributeError: 'apache_beam.coders.stream.ByteCountingOutputStream' object has no attribute 'buffer_size' [while running 'months with tornatoes'] {code} -- This message was sent by Atlassian JIRA (v6.3.15#6346)