Repository: beam Updated Branches: refs/heads/master f90558c34 -> 232869747
Improve DirectRunner performance by tuning BoundedReadEvaluator. Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d34145ad Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d34145ad Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d34145ad Branch: refs/heads/master Commit: d34145adf81dd50aa00d8e968fb2843bb624b23c Parents: f90558c Author: Younghee Kwon <younghee.k...@gmail.com> Authored: Mon Feb 6 21:49:49 2017 -0800 Committer: Younghee Kwon <younghee.k...@gmail.com> Committed: Mon Feb 6 21:49:49 2017 -0800 ---------------------------------------------------------------------- sdks/python/apache_beam/runners/direct/transform_evaluator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/d34145ad/sdks/python/apache_beam/runners/direct/transform_evaluator.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/direct/transform_evaluator.py b/sdks/python/apache_beam/runners/direct/transform_evaluator.py index 1700de6..6ae5697 100644 --- a/sdks/python/apache_beam/runners/direct/transform_evaluator.py +++ b/sdks/python/apache_beam/runners/direct/transform_evaluator.py @@ -175,7 +175,8 @@ class _TransformEvaluator(object): class _BoundedReadEvaluator(_TransformEvaluator): """TransformEvaluator for bounded Read transform.""" - MAX_ELEMENT_PER_BUNDLE = 100 + # After some benchmarks, 1000 was optimal among {100,1000,10000} + MAX_ELEMENT_PER_BUNDLE = 1000 def __init__(self, evaluation_context, applied_ptransform, input_committed_bundle, side_inputs, scoped_metrics_container):
