See
<https://builds.apache.org/job/beam_PostCommit_Python_Verify/1502/display/redirect?page=changes>
Changes:
[aljoscha.krettek] [BEAM-1689] Apply changes for Flink's StatefulDoFnRunner to
the primary
------------------------------------------
[...truncated 449.47 KB...]
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/Extract.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 744 bytes>",
"user_name": "write/Write/WriteImpl/Extract"
}
},
{
"kind": "CollectionToSingleton",
"name": "s15",
"properties": {
"display_data": [],
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name":
"write/Write/WriteImpl/ViewAsIterable(write|Write|WriteImpl|Extract.None)/CreatePCollectionView.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s14"
},
"user_name":
"write/Write/WriteImpl/ViewAsIterable(write|Write|WriteImpl|Extract.None)/CreatePCollectionView"
}
},
{
"kind": "ParallelDo",
"name": "s16",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_finalize_write"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {
"s15": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s15"
},
"s9": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s9"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type":
"FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/finalize_write.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "<string of 996 bytes>",
"user_name": "write/Write/WriteImpl/finalize_write"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2017-03-14T12:14:52.441257Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2017-03-14_05_14_52-13300760513112605405'
name: u'py-wordcount-1489489356'
projectId: u'apache-beam-testing'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2017-03-14_05_14_52-13300760513112605405]
root: INFO: To access the Dataflow monitoring console, please navigate to
https://console.developers.google.com/project/apache-beam-testing/dataflow/job/2017-03-14_05_14_52-13300760513112605405
root: INFO: Job 2017-03-14_05_14_52-13300760513112605405 is in state
JOB_STATE_RUNNING
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe0ed0:
2017-03-14T12:14:52.112Z: JOB_MESSAGE_WARNING: (b895cb375a9e8531): Setting the
number of workers (1) disables autoscaling for this job. If you are trying to
cap autoscaling, consider only setting max_num_workers. If you want to disable
autoscaling altogether, the documented way is to explicitly use
autoscalingAlgorithm=NONE.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe12d7:
2017-03-14T12:14:53.143Z: JOB_MESSAGE_DETAILED: (86320506aa5352b3): Checking
required Cloud APIs are enabled.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe14dd:
2017-03-14T12:14:53.661Z: JOB_MESSAGE_DEBUG: (86320506aa535529): Combiner
lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not
followed by a combiner.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe14df:
2017-03-14T12:14:53.663Z: JOB_MESSAGE_DEBUG: (86320506aa5355a7): Combiner
lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe14e2:
2017-03-14T12:14:53.666Z: JOB_MESSAGE_DETAILED: (86320506aa535625): Expanding
GroupByKey operations into optimizable parts.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe14e5:
2017-03-14T12:14:53.669Z: JOB_MESSAGE_DETAILED: (86320506aa5356a3): Lifting
ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe14ec:
2017-03-14T12:14:53.676Z: JOB_MESSAGE_DETAILED: (86320506aa53581d): Annotating
graph with Autotuner information.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe14f7:
2017-03-14T12:14:53.687Z: JOB_MESSAGE_DETAILED: (86320506aa535158): Fusing
adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe14fa:
2017-03-14T12:14:53.690Z: JOB_MESSAGE_DETAILED: (86320506aa5351d6): Fusing
consumer split into read/Read
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe14fd:
2017-03-14T12:14:53.693Z: JOB_MESSAGE_DETAILED: (86320506aa535254): Fusing
consumer group/Reify into pair_with_one
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1500:
2017-03-14T12:14:53.696Z: JOB_MESSAGE_DETAILED: (86320506aa5352d2): Fusing
consumer format into count
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1502:
2017-03-14T12:14:53.698Z: JOB_MESSAGE_DETAILED: (86320506aa535350): Fusing
consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into
write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1505:
2017-03-14T12:14:53.701Z: JOB_MESSAGE_DETAILED: (86320506aa5353ce): Fusing
consumer write/Write/WriteImpl/GroupByKey/Write into
write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe150b:
2017-03-14T12:14:53.707Z: JOB_MESSAGE_DETAILED: (86320506aa5354ca): Fusing
consumer write/Write/WriteImpl/Extract into
write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe150d:
2017-03-14T12:14:53.709Z: JOB_MESSAGE_DETAILED: (86320506aa535548): Fusing
consumer count into group/GroupByWindow
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1510:
2017-03-14T12:14:53.712Z: JOB_MESSAGE_DETAILED: (86320506aa5355c6): Fusing
consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into
write/Write/WriteImpl/Pair
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1512:
2017-03-14T12:14:53.714Z: JOB_MESSAGE_DETAILED: (86320506aa535644): Fusing
consumer write/Write/WriteImpl/GroupByKey/Reify into
write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1515:
2017-03-14T12:14:53.717Z: JOB_MESSAGE_DETAILED: (86320506aa5356c2): Fusing
consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1517:
2017-03-14T12:14:53.719Z: JOB_MESSAGE_DETAILED: (86320506aa535740): Fusing
consumer pair_with_one into split
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe151a:
2017-03-14T12:14:53.722Z: JOB_MESSAGE_DETAILED: (86320506aa5357be): Fusing
consumer group/GroupByWindow into group/Read
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe151c:
2017-03-14T12:14:53.724Z: JOB_MESSAGE_DETAILED: (86320506aa53583c): Fusing
consumer write/Write/WriteImpl/WriteBundles into format
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe151f:
2017-03-14T12:14:53.727Z: JOB_MESSAGE_DETAILED: (86320506aa5358ba): Fusing
consumer group/Write into group/Reify
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe156a:
2017-03-14T12:14:53.802Z: JOB_MESSAGE_DEBUG: (86320506aa53587a): Workflow
config is missing a default resource spec.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe156c:
2017-03-14T12:14:53.804Z: JOB_MESSAGE_DETAILED: (86320506aa5358f8): Adding
StepResource setup and teardown to workflow graph.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe156e:
2017-03-14T12:14:53.806Z: JOB_MESSAGE_DEBUG: (86320506aa535976): Adding
workflow start and stop steps.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1570:
2017-03-14T12:14:53.808Z: JOB_MESSAGE_DEBUG: (86320506aa5359f4): Assigning
stage ids.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1591:
2017-03-14T12:14:53.841Z: JOB_MESSAGE_DEBUG: (fb891257b0535b74): Executing wait
step start25
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1598:
2017-03-14T12:14:53.848Z: JOB_MESSAGE_BASIC: S01: (fb891257b0535abe): Executing
operation group/Create
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe159b:
2017-03-14T12:14:53.851Z: JOB_MESSAGE_DEBUG: (6fcfcacd71b839e2): Executing
operation write/Write/WriteImpl/DoOnce
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe15a4:
2017-03-14T12:14:53.860Z: JOB_MESSAGE_DEBUG: (6fcfcacd71b83a87): Value
"write/Write/WriteImpl/DoOnce.out" materialized.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe15ab:
2017-03-14T12:14:53.867Z: JOB_MESSAGE_BASIC: S04: (6fcfcacd71b8321f): Executing
operation write/Write/WriteImpl/InitializeWrite
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1664:
2017-03-14T12:14:54.052Z: JOB_MESSAGE_DEBUG: (43561ce1883587e0): Starting
worker pool setup.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1666:
2017-03-14T12:14:54.054Z: JOB_MESSAGE_BASIC: (43561ce188358e3e): Starting 1
workers...
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe1675:
2017-03-14T12:14:54.069Z: JOB_MESSAGE_DEBUG: (fb891257b05351ad): Value
"group/Session" materialized.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accbe167d:
2017-03-14T12:14:54.077Z: JOB_MESSAGE_BASIC: S02: (fb891257b0535b65): Executing
operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc0281e:
2017-03-14T12:17:09.662Z: JOB_MESSAGE_DETAILED: (de64d4e4bbb0a3c0): Workers
have started successfully.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc1f317:
2017-03-14T12:19:07.159Z: JOB_MESSAGE_DEBUG: (6fcfcacd71b83f0a): Value
"write/Write/WriteImpl/InitializeWrite.out" materialized.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc1f31f:
2017-03-14T12:19:07.167Z: JOB_MESSAGE_BASIC: S05: (6fcfcacd71b83faf): Executing
operation
write/Write/WriteImpl/ViewAsSingleton(write|Write|WriteImpl|InitializeWrite.None)/CreatePCollectionView
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc1f362:
2017-03-14T12:19:07.234Z: JOB_MESSAGE_DEBUG: (6fcfcacd71b83c20): Value
"write/Write/WriteImpl/ViewAsSingleton(write|Write|WriteImpl|InitializeWrite.None)/CreatePCollectionView.out"
materialized.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc1fd5c:
2017-03-14T12:19:09.788Z: JOB_MESSAGE_BASIC: S03: (fb891257b05359f9): Executing
operation group/Close
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc1fd73:
2017-03-14T12:19:09.811Z: JOB_MESSAGE_BASIC: S06: (fb891257b05350e8): Executing
operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc1fe25:
2017-03-14T12:19:09.989Z: JOB_MESSAGE_DEBUG: (fb891257b05357d7): Value
"write/Write/WriteImpl/GroupByKey/Session" materialized.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc1fe32:
2017-03-14T12:19:10.002Z: JOB_MESSAGE_BASIC: S07: (fb891257b0535458): Executing
operation
group/Read+group/GroupByWindow+count+format+write/Write/WriteImpl/WriteBundles+write/Write/WriteImpl/Pair+write/Write/WriteImpl/WindowInto(WindowIntoFn)+write/Write/WriteImpl/GroupByKey/Reify+write/Write/WriteImpl/GroupByKey/Write
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc224a8:
2017-03-14T12:19:19.848Z: JOB_MESSAGE_BASIC: S08: (fb891257b0535712): Executing
operation write/Write/WriteImpl/GroupByKey/Close
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc224bc:
2017-03-14T12:19:19.868Z: JOB_MESSAGE_BASIC: S09: (fb891257b05350ca): Executing
operation
write/Write/WriteImpl/GroupByKey/Read+write/Write/WriteImpl/GroupByKey/GroupByWindow+write/Write/WriteImpl/Extract
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc24be9:
2017-03-14T12:19:29.897Z: JOB_MESSAGE_DEBUG: (fb891257b0535a73): Value
"write/Write/WriteImpl/Extract.out" materialized.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc24bf1:
2017-03-14T12:19:29.905Z: JOB_MESSAGE_BASIC: S10: (fb891257b05356f4): Executing
operation
write/Write/WriteImpl/ViewAsIterable(write|Write|WriteImpl|Extract.None)/CreatePCollectionView
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc24c49:
2017-03-14T12:19:29.993Z: JOB_MESSAGE_DEBUG: (fb891257b0535851): Value
"write/Write/WriteImpl/ViewAsIterable(write|Write|WriteImpl|Extract.None)/CreatePCollectionView.out"
materialized.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc24c52:
2017-03-14T12:19:30.002Z: JOB_MESSAGE_BASIC: S11: (fb891257b05354d2): Executing
operation write/Write/WriteImpl/finalize_write
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc257da:
2017-03-14T12:19:32.954Z: JOB_MESSAGE_DEBUG: (fb891257b05354c3): Executing
success step success23
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc2581f:
2017-03-14T12:19:33.023Z: JOB_MESSAGE_DETAILED: (86320506aa5354a9): Cleaning up.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc25897:
2017-03-14T12:19:33.143Z: JOB_MESSAGE_DEBUG: (86320506aa535527): Starting
worker pool teardown.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc25899:
2017-03-14T12:19:33.145Z: JOB_MESSAGE_BASIC: (86320506aa5355a5): Stopping
worker pool...
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc39121:
2017-03-14T12:20:53.153Z: JOB_MESSAGE_BASIC: (86320506aa535e62): Worker pool
stopped.
root: INFO: 2017-03-14_05_14_52-13300760513112605405_0000015accc3913f:
2017-03-14T12:20:53.183Z: JOB_MESSAGE_DEBUG: (86320506aa535fdc): Tearing down
pending resources...
root: INFO: Job 2017-03-14_05_14_52-13300760513112605405 is in state
JOB_STATE_DONE
root: INFO: Read from given path
gs://temp-storage-for-end-to-end-tests/py-wordcount-cloud/output/py-wordcount-1489489356/results*-of-*,
3179 lines, checksum: a9bcb4acd65daf8f6a9ac5e026de7803cc09f662.
root: INFO: Read from given path
gs://temp-storage-for-end-to-end-tests/py-wordcount-cloud/output/py-wordcount-1489489356/results*-of-*,
4784 lines, checksum: 33535a832b7db6d78389759577d4ff495980b9c0.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 2 tests in 671.015s
FAILED (failures=1)
Build step 'Execute shell' marked build as failure
