See
<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/61/display/redirect?page=changes>
Changes:
[aaltay] Fix backlog bytes reported by CountingSource. (#5719)
------------------------------------------
[...truncated 145.64 KB...]
copying apache_beam/runners/portability/portable_runner.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/portability
copying apache_beam/runners/portability/portable_runner_test.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/portability
copying apache_beam/runners/portability/portable_stager.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/portability
copying apache_beam/runners/portability/portable_stager_test.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/portability
copying apache_beam/runners/portability/stager.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/portability
copying apache_beam/runners/portability/stager_test.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/portability
copying apache_beam/runners/test/__init__.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/test
copying apache_beam/runners/worker/__init__.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/bundle_processor.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/data_plane.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/data_plane_test.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/log_handler.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/log_handler_test.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/logger.pxd ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/logger.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/logger_test.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/opcounters.pxd ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/opcounters.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/opcounters_test.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/operation_specs.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/operations.pxd ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/operations.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/sdk_worker.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/sdk_worker_main.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/sdk_worker_main_test.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/sdk_worker_test.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/sideinputs.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/sideinputs_test.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/statesampler.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/statesampler_fast.pxd ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/statesampler_fast.pyx ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/statesampler_slow.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/statesampler_test.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/worker_id_interceptor.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/runners/worker/worker_id_interceptor_test.py ->
apache-beam-2.6.0.dev0/apache_beam/runners/worker
copying apache_beam/testing/__init__.py ->
apache-beam-2.6.0.dev0/apache_beam/testing
copying apache_beam/testing/pipeline_verifiers.py ->
apache-beam-2.6.0.dev0/apache_beam/testing
copying apache_beam/testing/pipeline_verifiers_test.py ->
apache-beam-2.6.0.dev0/apache_beam/testing
copying apache_beam/testing/test_pipeline.py ->
apache-beam-2.6.0.dev0/apache_beam/testing
copying apache_beam/testing/test_pipeline_test.py ->
apache-beam-2.6.0.dev0/apache_beam/testing
copying apache_beam/testing/test_stream.py ->
apache-beam-2.6.0.dev0/apache_beam/testing
copying apache_beam/testing/test_stream_test.py ->
apache-beam-2.6.0.dev0/apache_beam/testing
copying apache_beam/testing/test_utils.py ->
apache-beam-2.6.0.dev0/apache_beam/testing
copying apache_beam/testing/test_utils_test.py ->
apache-beam-2.6.0.dev0/apache_beam/testing
copying apache_beam/testing/util.py ->
apache-beam-2.6.0.dev0/apache_beam/testing
copying apache_beam/testing/util_test.py ->
apache-beam-2.6.0.dev0/apache_beam/testing
copying apache_beam/testing/data/standard_coders.yaml ->
apache-beam-2.6.0.dev0/apache_beam/testing/data
copying apache_beam/testing/data/trigger_transcripts.yaml ->
apache-beam-2.6.0.dev0/apache_beam/testing/data
copying apache_beam/tools/__init__.py ->
apache-beam-2.6.0.dev0/apache_beam/tools
copying apache_beam/tools/distribution_counter_microbenchmark.py ->
apache-beam-2.6.0.dev0/apache_beam/tools
copying apache_beam/tools/map_fn_microbenchmark.py ->
apache-beam-2.6.0.dev0/apache_beam/tools
copying apache_beam/tools/sideinput_microbenchmark.py ->
apache-beam-2.6.0.dev0/apache_beam/tools
copying apache_beam/tools/utils.py -> apache-beam-2.6.0.dev0/apache_beam/tools
copying apache_beam/transforms/__init__.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/combiners.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/combiners_test.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/core.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/create_source.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/create_test.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/cy_combiners.pxd ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/cy_combiners.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/cy_dataflow_distribution_counter.pxd ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/cy_dataflow_distribution_counter.pyx ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/dataflow_distribution_counter_test.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/display.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/display_test.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/ptransform.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/ptransform_test.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/py_dataflow_distribution_counter.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/sideinputs.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/sideinputs_test.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/timeutil.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/trigger.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/trigger_test.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/util.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/util_test.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/window.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/window_test.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/transforms/write_ptransform_test.py ->
apache-beam-2.6.0.dev0/apache_beam/transforms
copying apache_beam/typehints/__init__.py ->
apache-beam-2.6.0.dev0/apache_beam/typehints
copying apache_beam/typehints/decorators.py ->
apache-beam-2.6.0.dev0/apache_beam/typehints
copying apache_beam/typehints/native_type_compatibility.py ->
apache-beam-2.6.0.dev0/apache_beam/typehints
copying apache_beam/typehints/native_type_compatibility_test.py ->
apache-beam-2.6.0.dev0/apache_beam/typehints
copying apache_beam/typehints/opcodes.py ->
apache-beam-2.6.0.dev0/apache_beam/typehints
copying apache_beam/typehints/trivial_inference.py ->
apache-beam-2.6.0.dev0/apache_beam/typehints
copying apache_beam/typehints/trivial_inference_test.py ->
apache-beam-2.6.0.dev0/apache_beam/typehints
copying apache_beam/typehints/typecheck.py ->
apache-beam-2.6.0.dev0/apache_beam/typehints
copying apache_beam/typehints/typed_pipeline_test.py ->
apache-beam-2.6.0.dev0/apache_beam/typehints
copying apache_beam/typehints/typehints.py ->
apache-beam-2.6.0.dev0/apache_beam/typehints
copying apache_beam/typehints/typehints_test.py ->
apache-beam-2.6.0.dev0/apache_beam/typehints
copying apache_beam/utils/__init__.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/annotations.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/annotations_test.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/counters.pxd ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/counters.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/counters_test.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/plugin.py -> apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/processes.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/processes_test.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/profiler.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/proto_utils.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/retry.py -> apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/retry_test.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/timestamp.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/timestamp_test.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/urns.py -> apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/windowed_value.pxd ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/windowed_value.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
copying apache_beam/utils/windowed_value_test.py ->
apache-beam-2.6.0.dev0/apache_beam/utils
Writing apache-beam-2.6.0.dev0/setup.cfg
creating dist
Creating tar archive
removing 'apache-beam-2.6.0.dev0' (and everything under it)
SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
find dist/apache-beam-*.tar.gz
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
>>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST
python setup.py nosetests \
--attr ValidatesContainer \
--nocapture \
--processes=1 \
--process-timeout=900 \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--worker_harness_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1"
<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/container/local/lib/python2.7/site-packages/setuptools/dist.py>:398:
UserWarning: Normalizing '2.6.0.dev' to '2.6.0.dev0'
normalized_version,
running nosetests
running egg_info
<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/gen_protos.py>:49:
UserWarning: Installing grpcio-tools is recommended for development.
warnings.warn('Installing grpcio-tools is recommended for development.')
writing requirements to apache_beam.egg-info/requires.txt
writing apache_beam.egg-info/PKG-INFO
writing top-level names to apache_beam.egg-info/top_level.txt
writing dependency_links to apache_beam.egg-info/dependency_links.txt
writing entry points to apache_beam.egg-info/entry_points.txt
reading manifest file 'apache_beam.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
warning: no files found matching 'README.md'
warning: no files found matching 'NOTICE'
warning: no files found matching 'LICENSE'
writing manifest file 'apache_beam.egg-info/SOURCES.txt'
<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/gcp/gcsio.py>:176:
DeprecationWarning: object() takes no parameters
super(GcsIO, cls).__new__(cls, storage_client))
test_wordcount_fnapi_it (apache_beam.examples.wordcount_it_test.WordCountIT)
... ERROR
======================================================================
ERROR: test_wordcount_fnapi_it
(apache_beam.examples.wordcount_it_test.WordCountIT)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/examples/wordcount_it_test.py";,>
line 79, in test_wordcount_fnapi_it
on_success_matcher=PipelineStateMatcher()))
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/examples/wordcount_fnapi.py";,>
line 128, in run
result = p.run()
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/pipeline.py";,>
line 395, in run
self.to_runner_api(), self.runner, self._options).run(False)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/pipeline.py";,>
line 408, in run
return self.runner.run_pipeline(self)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/runners/dataflow/test_dataflow_runner.py";,>
line 48, in run_pipeline
self.result = super(TestDataflowRunner, self).run_pipeline(pipeline)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py";,>
line 350, in run_pipeline
super(DataflowRunner, self).run_pipeline(pipeline)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/runners/runner.py";,>
line 170, in run_pipeline
pipeline.visit(RunVisitor(self))
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/pipeline.py";,>
line 436, in visit
self._root_transform().visit(visitor, self, visited)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/pipeline.py";,>
line 791, in visit
part.visit(visitor, pipeline, visited)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/pipeline.py";,>
line 791, in visit
part.visit(visitor, pipeline, visited)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/pipeline.py";,>
line 791, in visit
part.visit(visitor, pipeline, visited)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/pipeline.py";,>
line 794, in visit
visitor.visit_transform(self)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/runners/runner.py";,>
line 165, in visit_transform
self.runner.run_transform(transform_node)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/runners/runner.py";,>
line 208, in run_transform
return m(transform_node)
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py";,>
line 586, in run_ParDo
input_step = self._cache.get_pvalue(transform_node.inputs[0])
File
"<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/runners/runner.py";,>
line 286, in get_pvalue
return self._cache[self.key(pvalue)]
KeyError: (u'read/Read/Impulse', None)
-------------------- >> begin captured logging << --------------------
root: DEBUG: Connecting using Google Application Default Credentials.
oauth2client.transport: INFO: Attempting refresh to obtain initial access_token
root: DEBUG: Connecting using Google Application Default Credentials.
oauth2client.transport: INFO: Attempting refresh to obtain initial access_token
root: ERROR: Error while visiting read/Read/Split
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML:
<https://builds.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 1.689s
FAILED (errors=1)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user apill...@google.com
Not sending mail to unregistered user katarzyna.kucharc...@polidea.com
Not sending mail to unregistered user mig...@google.com
Not sending mail to unregistered user git...@alasdairhodge.co.uk
Not sending mail to unregistered user marek.simu...@firma.seznam.cz
Not sending mail to unregistered user lukasz.gaj...@polidea.com
Not sending mail to unregistered user sourabhba...@google.com
Not sending mail to unregistered user merge...@apache.org
Not sending mail to unregistered user mari...@mariagh.svl.corp.google.com
Not sending mail to unregistered user thomas.hai...@practiceinsight.io
Not sending mail to unregistered user axelm...@gmail.com
Not sending mail to unregistered user ddonne...@google.com
Not sending mail to unregistered user eh...@google.com
Not sending mail to unregistered user
re...@relax-macbookpro2.roam.corp.google.com
Not sending mail to unregistered user devon.meun...@shopify.com
Not sending mail to unregistered user timrobertson...@gmail.com
Not sending mail to unregistered user kirpic...@google.com
Not sending mail to unregistered user cademarkeg...@gmail.com
Not sending mail to unregistered user ekirpic...@gmail.com
Not sending mail to unregistered user basti...@alibaba-inc.com
Not sending mail to unregistered user m...@nrd.io
Not sending mail to unregistered user ro...@frantil.com
Not sending mail to unregistered user jiang...@gmail.com
Not sending mail to unregistered user valen...@google.com
Not sending mail to unregistered user aal...@gmail.com
Not sending mail to unregistered user a...@paypal.com
Not sending mail to unregistered user jamal...@gmail.com
Not sending mail to unregistered user terrence...@tencent.com
Not sending mail to unregistered user had...@google.com
Not sending mail to unregistered user meliss...@google.com
Not sending mail to unregistered user f...@google.com
Not sending mail to unregistered user k...@google.com
Not sending mail to unregistered user pger...@us.ibm.com
Not sending mail to unregistered user jasonkus...@google.com
Not sending mail to unregistered user aljoscha.kret...@gmail.com
Not sending mail to unregistered user aromanenko....@gmail.com
Not sending mail to unregistered user mair...@google.com
Not sending mail to unregistered user taro.mu...@gmail.com
Not sending mail to unregistered user sweg...@google.com
Not sending mail to unregistered user ankurgoe...@gmail.com
Not sending mail to unregistered user wnsdud1...@gmail.com
Not sending mail to unregistered user pei...@gmail.com
Not sending mail to unregistered user s...@google.com
