[BEAM-1340] Add __all__ tags to modules in package apache_beam/transforms
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/6d02da03 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/6d02da03 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/6d02da03 Branch: refs/heads/release-2.0.0 Commit: 6d02da03981f08037538e63e9246efef63a36ea0 Parents: 0c784f9 Author: Charles Chen <[email protected]> Authored: Wed May 10 23:06:36 2017 -0700 Committer: Ahmet Altay <[email protected]> Committed: Thu May 11 16:20:36 2017 -0700 ---------------------------------------------------------------------- .../examples/snippets/snippets_test.py | 2 +- .../runners/dataflow/dataflow_runner.py | 2 +- .../runners/dataflow/dataflow_runner_test.py | 5 +++-- sdks/python/apache_beam/transforms/core.py | 20 ++++++++++++++++++++ .../apache_beam/transforms/cy_combiners.py | 5 ++++- .../python/apache_beam/transforms/ptransform.py | 9 +++++++++ .../apache_beam/transforms/ptransform_test.py | 11 ++++++----- .../python/apache_beam/transforms/sideinputs.py | 2 ++ sdks/python/apache_beam/transforms/timeutil.py | 5 +++++ sdks/python/apache_beam/transforms/trigger.py | 14 ++++++++++++++ sdks/python/apache_beam/transforms/window.py | 15 +++++++++++++++ 11 files changed, 80 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/6d02da03/sdks/python/apache_beam/examples/snippets/snippets_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py index 211da24..37cd470 100644 --- a/sdks/python/apache_beam/examples/snippets/snippets_test.py +++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py @@ -892,7 +892,7 @@ class CombineTest(unittest.TestCase): unix_timestamp = extract_timestamp_from_log_entry(element) # Wrap and emit the current entry and new timestamp in a # TimestampedValue. - yield beam.TimestampedValue(element, unix_timestamp) + yield beam.window.TimestampedValue(element, unix_timestamp) timestamped_items = items | 'timestamp' >> beam.ParDo(AddTimestampDoFn()) # [END setting_timestamp] http://git-wip-us.apache.org/repos/asf/beam/blob/6d02da03/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index 3d8437c..da8de9d 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -169,7 +169,7 @@ class DataflowRunner(PipelineRunner): def visit_transform(self, transform_node): # Imported here to avoid circular dependencies. # pylint: disable=wrong-import-order, wrong-import-position - from apache_beam import GroupByKey, GroupByKeyOnly + from apache_beam.transforms.core import GroupByKey, GroupByKeyOnly if isinstance(transform_node.transform, (GroupByKey, GroupByKeyOnly)): pcoll = transform_node.inputs[0] input_type = pcoll.element_type http://git-wip-us.apache.org/repos/asf/beam/blob/6d02da03/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py index b61a683..ac9b028 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner_test.py @@ -37,6 +37,7 @@ from apache_beam.runners.dataflow.dataflow_runner import DataflowRuntimeExceptio from apache_beam.runners.dataflow.internal.clients import dataflow as dataflow_api from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.transforms.display import DisplayDataItem +from apache_beam.transforms.core import GroupByKeyOnly from apache_beam.typehints import typehints # Protect against environments where apitools library is not available. @@ -184,7 +185,7 @@ class DataflowRunnerTest(unittest.TestCase): pcoll1 = PCollection(p) pcoll2 = PCollection(p) pcoll3 = PCollection(p) - for transform in [beam.GroupByKeyOnly(), beam.GroupByKey()]: + for transform in [GroupByKeyOnly(), beam.GroupByKey()]: pcoll1.element_type = None pcoll2.element_type = typehints.Any pcoll3.element_type = typehints.KV[typehints.Any, typehints.Any] @@ -198,7 +199,7 @@ class DataflowRunnerTest(unittest.TestCase): p = TestPipeline() pcoll1 = PCollection(p) pcoll2 = PCollection(p) - for transform in [beam.GroupByKeyOnly(), beam.GroupByKey()]: + for transform in [GroupByKeyOnly(), beam.GroupByKey()]: pcoll1.element_type = typehints.TupleSequenceConstraint pcoll2.element_type = typehints.Set err_msg = "Input to GroupByKey must be of Tuple or Any type" http://git-wip-us.apache.org/repos/asf/beam/blob/6d02da03/sdks/python/apache_beam/transforms/core.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py index d42115c..a1964cf 100644 --- a/sdks/python/apache_beam/transforms/core.py +++ b/sdks/python/apache_beam/transforms/core.py @@ -53,6 +53,26 @@ from apache_beam.utils import urns from apache_beam.options.pipeline_options import TypeOptions +__all__ = [ + 'DoFn', + 'CombineFn', + 'PartitionFn', + 'ParDo', + 'FlatMap', + 'Map', + 'Filter', + 'CombineGlobally', + 'CombinePerKey', + 'CombineValues', + 'GroupByKey', + 'Partition', + 'Windowing', + 'WindowInto', + 'Flatten', + 'Create', + ] + + # Type variables T = typehints.TypeVariable('T') K = typehints.TypeVariable('K') http://git-wip-us.apache.org/repos/asf/beam/blob/6d02da03/sdks/python/apache_beam/transforms/cy_combiners.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/cy_combiners.py b/sdks/python/apache_beam/transforms/cy_combiners.py index f824870..84aee21 100644 --- a/sdks/python/apache_beam/transforms/cy_combiners.py +++ b/sdks/python/apache_beam/transforms/cy_combiners.py @@ -15,7 +15,10 @@ # limitations under the License. # -"""A library of basic cythonized CombineFn subclasses.""" +"""A library of basic cythonized CombineFn subclasses. + +For internal use only; no backwards-compatibility guarantees. +""" from __future__ import absolute_import http://git-wip-us.apache.org/repos/asf/beam/blob/6d02da03/sdks/python/apache_beam/transforms/ptransform.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py index fb79b19..d1f9835 100644 --- a/sdks/python/apache_beam/transforms/ptransform.py +++ b/sdks/python/apache_beam/transforms/ptransform.py @@ -60,6 +60,13 @@ from apache_beam.utils import proto_utils from apache_beam.utils import urns +__all__ = [ + 'PTransform', + 'ptransform_fn', + 'label_from_callable', + ] + + class _PValueishTransform(object): """Visitor for PValueish objects. @@ -639,6 +646,8 @@ class CallablePTransform(PTransform): def ptransform_fn(fn): """A decorator for a function-based PTransform. + Experimental; no backwards-compatibility guarantees. + Args: fn: A function implementing a custom PTransform. http://git-wip-us.apache.org/repos/asf/beam/blob/6d02da03/sdks/python/apache_beam/transforms/ptransform_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py index 137992d..3320d79 100644 --- a/sdks/python/apache_beam/transforms/ptransform_test.py +++ b/sdks/python/apache_beam/transforms/ptransform_test.py @@ -34,6 +34,7 @@ from apache_beam.options.pipeline_options import TypeOptions import apache_beam.pvalue as pvalue from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.transforms import window +from apache_beam.transforms.core import GroupByKeyOnly import apache_beam.transforms.combiners as combine from apache_beam.transforms.display import DisplayData, DisplayDataItem from apache_beam.transforms.ptransform import PTransform @@ -579,7 +580,7 @@ class PTransformTest(unittest.TestCase): pipeline = TestPipeline() pcolls = pipeline | 'A' >> beam.Create(['a', 'b', 'f']) with self.assertRaises(typehints.TypeCheckError) as cm: - pcolls | 'D' >> beam.GroupByKeyOnly() + pcolls | 'D' >> GroupByKeyOnly() pipeline.run() expected_error_prefix = ('Input type hint violation at D: expected ' @@ -1087,7 +1088,7 @@ class PTransformTypeCheckTestCase(TypeHintTestCase): | 'Str' >> beam.Create(['t', 'e', 's', 't']).with_output_types(str) | ('Pair' >> beam.Map(lambda x: (x, ord(x))) .with_output_types(typehints.KV[str, str])) - | beam.GroupByKeyOnly()) + | GroupByKeyOnly()) # Output type should correctly be deduced. # GBK-only should deduce that KV[A, B] is turned into KV[A, Iterable[B]]. @@ -1111,7 +1112,7 @@ class PTransformTypeCheckTestCase(TypeHintTestCase): with self.assertRaises(typehints.TypeCheckError) as e: (self.p | beam.Create([1, 2, 3]).with_output_types(int) - | 'F' >> beam.GroupByKeyOnly()) + | 'F' >> GroupByKeyOnly()) self.assertEqual("Input type hint violation at F: " "expected Tuple[TypeVariable[K], TypeVariable[V]], " @@ -1154,7 +1155,7 @@ class PTransformTypeCheckTestCase(TypeHintTestCase): (self.p | 'Nums' >> beam.Create(range(5)).with_output_types(int) | 'ModDup' >> beam.Map(lambda x: (x % 2, x)) - | beam.GroupByKeyOnly()) + | GroupByKeyOnly()) self.assertEqual('Pipeline type checking is enabled, however no output ' 'type-hint was found for the PTransform ' @@ -1977,7 +1978,7 @@ class PTransformTypeCheckTestCase(TypeHintTestCase): def test_gbk_type_inference(self): self.assertEqual( typehints.Tuple[str, typehints.Iterable[int]], - beam.core.GroupByKeyOnly().infer_output_type(typehints.KV[str, int])) + GroupByKeyOnly().infer_output_type(typehints.KV[str, int])) def test_pipeline_inference(self): created = self.p | beam.Create(['a', 'b', 'c']) http://git-wip-us.apache.org/repos/asf/beam/blob/6d02da03/sdks/python/apache_beam/transforms/sideinputs.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/sideinputs.py b/sdks/python/apache_beam/transforms/sideinputs.py index 6ba5311..f10cb92 100644 --- a/sdks/python/apache_beam/transforms/sideinputs.py +++ b/sdks/python/apache_beam/transforms/sideinputs.py @@ -17,6 +17,8 @@ """Internal side input transforms and implementations. +For internal use only; no backwards-compatibility guarantees. + Important: this module is an implementation detail and should not be used directly by pipeline writers. Instead, users should use the helper methods AsSingleton, AsIter, AsList and AsDict in apache_beam.pvalue. http://git-wip-us.apache.org/repos/asf/beam/blob/6d02da03/sdks/python/apache_beam/transforms/timeutil.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/timeutil.py b/sdks/python/apache_beam/transforms/timeutil.py index ba4ef36..c0f9198 100644 --- a/sdks/python/apache_beam/transforms/timeutil.py +++ b/sdks/python/apache_beam/transforms/timeutil.py @@ -23,6 +23,11 @@ from abc import ABCMeta from abc import abstractmethod +__all__ = [ + 'TimeDomain', + ] + + class TimeDomain(object): """Time domain for streaming timers.""" http://git-wip-us.apache.org/repos/asf/beam/blob/6d02da03/sdks/python/apache_beam/transforms/trigger.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py index 2cb7ce3..7de2f85 100644 --- a/sdks/python/apache_beam/transforms/trigger.py +++ b/sdks/python/apache_beam/transforms/trigger.py @@ -38,6 +38,20 @@ from apache_beam.utils.timestamp import MAX_TIMESTAMP from apache_beam.utils.timestamp import MIN_TIMESTAMP +__all__ = [ + 'AccumulationMode', + 'TriggerFn', + 'DefaultTrigger', + 'AfterWatermark', + 'AfterCount', + 'Repeatedly', + 'AfterAny', + 'AfterAll', + 'AfterEach', + 'OrFinally', + ] + + class AccumulationMode(object): """Controls what to do with data when a trigger fires multiple times. """ http://git-wip-us.apache.org/repos/asf/beam/blob/6d02da03/sdks/python/apache_beam/transforms/window.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py index 6d0db3a..94187e0 100644 --- a/sdks/python/apache_beam/transforms/window.py +++ b/sdks/python/apache_beam/transforms/window.py @@ -65,6 +65,21 @@ from apache_beam.utils.timestamp import Timestamp from apache_beam.utils.windowed_value import WindowedValue +__all__ = [ + 'TimestampCombiner', + 'WindowFn', + 'BoundedWindow', + 'IntervalWindow', + 'TimestampedValue', + 'GlobalWindow', + 'NonMergingWindowFn', + 'GlobalWindows', + 'FixedWindows', + 'SlidingWindows', + 'Sessions', + ] + + # TODO(ccy): revisit naming and semantics once Java Apache Beam finalizes their # behavior. class TimestampCombiner(object):
