[BEAM-1340] Adds __all__ tags to classes in package apache_beam/io.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0c784f95 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0c784f95 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0c784f95 Branch: refs/heads/release-2.0.0 Commit: 0c784f958f11eb0e448cc448d2b9d36fd90e9972 Parents: dec27d8 Author: [email protected] <[email protected]> Authored: Thu May 11 11:46:46 2017 -0700 Committer: Ahmet Altay <[email protected]> Committed: Thu May 11 16:20:36 2017 -0700 ---------------------------------------------------------------------- sdks/python/apache_beam/io/concat_source.py | 4 +++- sdks/python/apache_beam/io/filebasedsource.py | 2 ++ sdks/python/apache_beam/io/fileio.py | 7 +++++++ sdks/python/apache_beam/io/filesystem.py | 3 +++ sdks/python/apache_beam/io/filesystems.py | 2 ++ sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 2 ++ sdks/python/apache_beam/io/gcp/gcsio.py | 3 +++ sdks/python/apache_beam/io/gcp/pubsub.py | 2 ++ sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py | 3 +++ sdks/python/apache_beam/io/iobase.py | 2 ++ sdks/python/apache_beam/io/localfilesystem.py | 2 ++ sdks/python/apache_beam/io/range_trackers.py | 7 ++++++- sdks/python/apache_beam/io/source_test_utils.py | 8 ++++++++ 13 files changed, 45 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/concat_source.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/concat_source.py b/sdks/python/apache_beam/io/concat_source.py index dfd1695..56c4cca 100644 --- a/sdks/python/apache_beam/io/concat_source.py +++ b/sdks/python/apache_beam/io/concat_source.py @@ -15,7 +15,9 @@ # limitations under the License. # -"""Concat Source, which reads the union of several other sources. +"""For internal use only; no backwards-compatibility guarantees. + +Concat Source, which reads the union of several other sources. """ import bisect http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/filebasedsource.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/filebasedsource.py b/sdks/python/apache_beam/io/filebasedsource.py index 215e015..bb9efc4 100644 --- a/sdks/python/apache_beam/io/filebasedsource.py +++ b/sdks/python/apache_beam/io/filebasedsource.py @@ -38,6 +38,8 @@ from apache_beam.options.value_provider import check_accessible MAX_NUM_THREADS_FOR_SIZE_ESTIMATION = 25 +__all__ = ['FileBasedSource'] + class FileBasedSource(iobase.BoundedSource): """A ``BoundedSource`` for reading a file glob of a given type.""" http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/fileio.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py index ca3a759..aa18093 100644 --- a/sdks/python/apache_beam/io/fileio.py +++ b/sdks/python/apache_beam/io/fileio.py @@ -37,6 +37,8 @@ from apache_beam.options.value_provider import check_accessible DEFAULT_SHARD_NAME_TEMPLATE = '-SSSSS-of-NNNNN' +__all__ = ['FileBasedSink'] + class FileSink(iobase.Sink): """A sink to a GCS or local files. @@ -280,6 +282,11 @@ class FileSink(iobase.Sink): return type(self) == type(other) and self.__dict__ == other.__dict__ +# Using FileBasedSink for the public API to be symmetric with FileBasedSource. +# TODO: move code from FileSink to here and delete that class. +FileBasedSink = FileSink + + class FileSinkWriter(iobase.Writer): """The writer for FileSink. """ http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/filesystem.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/filesystem.py b/sdks/python/apache_beam/io/filesystem.py index 3d35f3e..db6a1d0 100644 --- a/sdks/python/apache_beam/io/filesystem.py +++ b/sdks/python/apache_beam/io/filesystem.py @@ -30,6 +30,9 @@ logger = logging.getLogger(__name__) DEFAULT_READ_BUFFER_SIZE = 16 * 1024 * 1024 +__all__ = ['CompressionTypes', 'CompressedFile', 'FileMetadata', 'FileSystem', + 'MatchResult'] + class CompressionTypes(object): """Enum-like class representing known compression types.""" http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/filesystems.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/filesystems.py b/sdks/python/apache_beam/io/filesystems.py index 29f0644..e039686 100644 --- a/sdks/python/apache_beam/io/filesystems.py +++ b/sdks/python/apache_beam/io/filesystems.py @@ -33,6 +33,8 @@ except ImportError: pass # pylint: enable=wrong-import-position, unused-import +__all__ = ['FileSystems'] + class FileSystems(object): """A class that defines the functions that can be performed on a filesystem. http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/gcp/gcsfilesystem.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem.py index dc71fce..ce8b5e6 100644 --- a/sdks/python/apache_beam/io/gcp/gcsfilesystem.py +++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem.py @@ -26,6 +26,8 @@ from apache_beam.io.filesystem import FileSystem from apache_beam.io.filesystem import MatchResult from apache_beam.io.gcp import gcsio +__all__ = ['GCSFileSystem'] + class GCSFileSystem(FileSystem): """A GCS ``FileSystem`` implementation for accessing files on GCS. http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/gcp/gcsio.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py index c76c99d..774ee54 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio.py +++ b/sdks/python/apache_beam/io/gcp/gcsio.py @@ -34,6 +34,9 @@ import traceback from apache_beam.utils import retry +__all__ = ['GcsIO'] + + # Issue a friendlier error message if the storage library is not available. # TODO(silviuc): Remove this guard when storage is available everywhere. try: http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/gcp/pubsub.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py index efc628d..103fce0 100644 --- a/sdks/python/apache_beam/io/gcp/pubsub.py +++ b/sdks/python/apache_beam/io/gcp/pubsub.py @@ -26,6 +26,8 @@ from apache_beam import coders from apache_beam.runners.dataflow.native_io import iobase as dataflow_io from apache_beam.transforms.display import DisplayDataItem +__all__ = ['PubSubSink', 'PubSubSource'] + class PubSubSource(dataflow_io.NativeSource): """Source for reading from a given Cloud Pub/Sub topic. http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py index f42b70f..844cbc5 100644 --- a/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py +++ b/sdks/python/apache_beam/io/gcp/tests/bigquery_matcher.py @@ -24,6 +24,9 @@ from hamcrest.core.base_matcher import BaseMatcher from apache_beam.testing.test_utils import compute_hash from apache_beam.utils import retry +__all__ = ['BigqueryMatcher'] + + # Protect against environments where bigquery library is not available. # pylint: disable=wrong-import-order, wrong-import-position try: http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/iobase.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/iobase.py b/sdks/python/apache_beam/io/iobase.py index e8ffb72..a80b12f 100644 --- a/sdks/python/apache_beam/io/iobase.py +++ b/sdks/python/apache_beam/io/iobase.py @@ -46,6 +46,8 @@ from apache_beam.transforms.display import HasDisplayData from apache_beam.transforms.display import DisplayDataItem from apache_beam.utils.windowed_value import WindowedValue +__all__ = ['BoundedSource', 'RangeTracker', 'Read', 'Sink', 'Write', 'Writer'] + # Encapsulates information about a bundle of a source generated when method # BoundedSource.split() is invoked. http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/localfilesystem.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/localfilesystem.py b/sdks/python/apache_beam/io/localfilesystem.py index c670704..b08ac49 100644 --- a/sdks/python/apache_beam/io/localfilesystem.py +++ b/sdks/python/apache_beam/io/localfilesystem.py @@ -29,6 +29,8 @@ from apache_beam.io.filesystem import FileMetadata from apache_beam.io.filesystem import FileSystem from apache_beam.io.filesystem import MatchResult +__all__ = ['LocalFileSystem'] + class LocalFileSystem(FileSystem): """A Local ``FileSystem`` implementation for accessing files on disk. http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/range_trackers.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/range_trackers.py b/sdks/python/apache_beam/io/range_trackers.py index 000df81..9cb36e7 100644 --- a/sdks/python/apache_beam/io/range_trackers.py +++ b/sdks/python/apache_beam/io/range_trackers.py @@ -24,6 +24,9 @@ import threading from apache_beam.io import iobase +__all__ = ['OffsetRangeTracker', 'LexicographicKeyRangeTracker', + 'OrderedPositionRangeTracker', 'UnsplittableRangeTracker'] + class OffsetRangeTracker(iobase.RangeTracker): """A 'RangeTracker' for non-negative positions of type 'long'.""" @@ -191,7 +194,9 @@ class OffsetRangeTracker(iobase.RangeTracker): class GroupedShuffleRangeTracker(iobase.RangeTracker): - """A 'RangeTracker' for positions used by'GroupedShuffleReader'. + """For internal use only; no backwards-compatibility guarantees. + + A 'RangeTracker' for positions used by'GroupedShuffleReader'. These positions roughly correspond to hashes of keys. In case of hash collisions, multiple groups can have the same position. In that case, the http://git-wip-us.apache.org/repos/asf/beam/blob/0c784f95/sdks/python/apache_beam/io/source_test_utils.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/source_test_utils.py b/sdks/python/apache_beam/io/source_test_utils.py index edb6409..a144a8a 100644 --- a/sdks/python/apache_beam/io/source_test_utils.py +++ b/sdks/python/apache_beam/io/source_test_utils.py @@ -52,6 +52,14 @@ import weakref from multiprocessing.pool import ThreadPool from apache_beam.io import iobase +__all__ = ['read_from_source', 'assert_sources_equal_reference_source', + 'assert_reentrant_reads_succeed', + 'assert_split_at_fraction_behavior', + 'assert_split_at_fraction_binary', + 'assert_split_at_fraction_exhaustive', + 'assert_split_at_fraction_fails', + 'assert_split_at_fraction_succeeds_and_consistent'] + class ExpectedSplitOutcome(object): MUST_SUCCEED_AND_BE_CONSISTENT = 1
