Repository: beam Updated Branches: refs/heads/master 646caf255 -> eebff9089
Move Runner API protos to portability/api Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/7e4e51fd Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/7e4e51fd Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/7e4e51fd Branch: refs/heads/master Commit: 7e4e51fda633d208c4cc5e88182e5db16156f2cb Parents: 646caf2 Author: Charles Chen <c...@google.com> Authored: Tue Jun 13 15:03:15 2017 -0700 Committer: Charles Chen <c...@google.com> Committed: Tue Jun 13 15:07:16 2017 -0700 ---------------------------------------------------------------------- .gitignore | 2 +- sdks/python/apache_beam/coders/coders.py | 2 +- sdks/python/apache_beam/pipeline.py | 4 ++-- .../apache_beam/portability/api/__init__.py | 21 ++++++++++++++++++++ .../apache_beam/portability/runners/__init__.py | 18 ----------------- .../portability/runners/api/__init__.py | 21 -------------------- sdks/python/apache_beam/pvalue.py | 2 +- .../runners/dataflow/dataflow_runner.py | 4 ++-- .../apache_beam/runners/pipeline_context.py | 2 +- .../runners/portability/fn_api_runner.py | 2 +- .../apache_beam/runners/worker/data_plane.py | 2 +- .../runners/worker/data_plane_test.py | 2 +- .../apache_beam/runners/worker/log_handler.py | 2 +- .../runners/worker/log_handler_test.py | 2 +- .../apache_beam/runners/worker/sdk_worker.py | 2 +- .../runners/worker/sdk_worker_main.py | 2 +- .../runners/worker/sdk_worker_test.py | 2 +- sdks/python/apache_beam/transforms/core.py | 2 +- .../python/apache_beam/transforms/ptransform.py | 2 +- sdks/python/apache_beam/transforms/trigger.py | 2 +- sdks/python/apache_beam/transforms/window.py | 4 ++-- sdks/python/apache_beam/utils/urns.py | 2 +- sdks/python/gen_protos.py | 2 +- sdks/python/run_pylint.sh | 2 +- 24 files changed, 45 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 631d7f3..36c5cc8 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,7 @@ sdks/python/**/*.egg sdks/python/LICENSE sdks/python/NOTICE sdks/python/README.md -sdks/python/apache_beam/portability/runners/api/*pb2*.* +sdks/python/apache_beam/portability/api/*pb2*.* # Ignore IntelliJ files. .idea/ http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/coders/coders.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py index 1be1f3c..c56ef52 100644 --- a/sdks/python/apache_beam/coders/coders.py +++ b/sdks/python/apache_beam/coders/coders.py @@ -25,7 +25,7 @@ import cPickle as pickle import google.protobuf from apache_beam.coders import coder_impl -from apache_beam.portability.runners.api import beam_runner_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.utils import urns from apache_beam.utils import proto_utils http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/pipeline.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py index 05715d7..ab77956 100644 --- a/sdks/python/apache_beam/pipeline.py +++ b/sdks/python/apache_beam/pipeline.py @@ -492,7 +492,7 @@ class Pipeline(object): def to_runner_api(self): """For internal use only; no backwards-compatibility guarantees.""" from apache_beam.runners import pipeline_context - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 context = pipeline_context.PipelineContext() # Mutates context; placing inline would force dependence on # argument evaluation order. @@ -692,7 +692,7 @@ class AppliedPTransform(object): if isinstance(output, pvalue.PCollection)} def to_runner_api(self, context): - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 def transform_to_runner_api(transform, context): if transform is None: http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/portability/api/__init__.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/portability/api/__init__.py b/sdks/python/apache_beam/portability/api/__init__.py new file mode 100644 index 0000000..2750859 --- /dev/null +++ b/sdks/python/apache_beam/portability/api/__init__.py @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""For internal use only; no backwards-compatibility guarantees. + +Automatically generated when running setup.py sdist or build[_py]. +""" http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/portability/runners/__init__.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/portability/runners/__init__.py b/sdks/python/apache_beam/portability/runners/__init__.py deleted file mode 100644 index 0bce5d6..0000000 --- a/sdks/python/apache_beam/portability/runners/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""For internal use only; no backwards-compatibility guarantees.""" http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/portability/runners/api/__init__.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/portability/runners/api/__init__.py b/sdks/python/apache_beam/portability/runners/api/__init__.py deleted file mode 100644 index 2750859..0000000 --- a/sdks/python/apache_beam/portability/runners/api/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""For internal use only; no backwards-compatibility guarantees. - -Automatically generated when running setup.py sdist or build[_py]. -""" http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/pvalue.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/pvalue.py b/sdks/python/apache_beam/pvalue.py index 8a774c4..34a483e 100644 --- a/sdks/python/apache_beam/pvalue.py +++ b/sdks/python/apache_beam/pvalue.py @@ -128,7 +128,7 @@ class PCollection(PValue): return _InvalidUnpickledPCollection, () def to_runner_api(self, context): - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.internal import pickler return beam_runner_api_pb2.PCollection( unique_name='%d%s.%s' % ( http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index a6cc25d..d6944b2 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -732,7 +732,7 @@ class DataflowRunner(PipelineRunner): @classmethod def serialize_windowing_strategy(cls, windowing): from apache_beam.runners import pipeline_context - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 context = pipeline_context.PipelineContext() windowing_proto = windowing.to_runner_api(context) return cls.byte_array_to_json_string( @@ -745,7 +745,7 @@ class DataflowRunner(PipelineRunner): # Imported here to avoid circular dependencies. # pylint: disable=wrong-import-order, wrong-import-position from apache_beam.runners import pipeline_context - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.transforms.core import Windowing proto = beam_runner_api_pb2.MessageWithComponents() proto.ParseFromString(cls.json_string_to_byte_array(serialized_data)) http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/runners/pipeline_context.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/pipeline_context.py b/sdks/python/apache_beam/runners/pipeline_context.py index 1330c39..e212abf 100644 --- a/sdks/python/apache_beam/runners/pipeline_context.py +++ b/sdks/python/apache_beam/runners/pipeline_context.py @@ -24,7 +24,7 @@ For internal use only; no backwards-compatibility guarantees. from apache_beam import pipeline from apache_beam import pvalue from apache_beam import coders -from apache_beam.portability.runners.api import beam_runner_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.transforms import core http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/runners/portability/fn_api_runner.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner.py b/sdks/python/apache_beam/runners/portability/fn_api_runner.py index 8c213ad..90764f4 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner.py @@ -33,7 +33,7 @@ from apache_beam.coders.coder_impl import create_OutputStream from apache_beam.internal import pickler from apache_beam.io import iobase from apache_beam.transforms.window import GlobalWindows -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.portability import maptask_executor_runner from apache_beam.runners.worker import data_plane from apache_beam.runners.worker import operation_specs http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/runners/worker/data_plane.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/worker/data_plane.py b/sdks/python/apache_beam/runners/worker/data_plane.py index 734ee9c..bc981a8 100644 --- a/sdks/python/apache_beam/runners/worker/data_plane.py +++ b/sdks/python/apache_beam/runners/worker/data_plane.py @@ -28,7 +28,7 @@ import Queue as queue import threading from apache_beam.coders import coder_impl -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 import grpc # This module is experimental. No backwards-compatibility guarantees. http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/runners/worker/data_plane_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/worker/data_plane_test.py b/sdks/python/apache_beam/runners/worker/data_plane_test.py index a2b31e8..360468a 100644 --- a/sdks/python/apache_beam/runners/worker/data_plane_test.py +++ b/sdks/python/apache_beam/runners/worker/data_plane_test.py @@ -29,7 +29,7 @@ import unittest from concurrent import futures import grpc -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.worker import data_plane http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/runners/worker/log_handler.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/worker/log_handler.py b/sdks/python/apache_beam/runners/worker/log_handler.py index dca0e4b..b8f6352 100644 --- a/sdks/python/apache_beam/runners/worker/log_handler.py +++ b/sdks/python/apache_beam/runners/worker/log_handler.py @@ -21,7 +21,7 @@ import math import Queue as queue import threading -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 import grpc # This module is experimental. No backwards-compatibility guarantees. http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/runners/worker/log_handler_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/worker/log_handler_test.py b/sdks/python/apache_beam/runners/worker/log_handler_test.py index 6dd018f..2256bb5 100644 --- a/sdks/python/apache_beam/runners/worker/log_handler_test.py +++ b/sdks/python/apache_beam/runners/worker/log_handler_test.py @@ -22,7 +22,7 @@ import unittest from concurrent import futures import grpc -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.worker import log_handler http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/runners/worker/sdk_worker.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker.py b/sdks/python/apache_beam/runners/worker/sdk_worker.py index dc4f5c2..f662538 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker.py @@ -38,7 +38,7 @@ from apache_beam.internal import pickler from apache_beam.io import iobase from apache_beam.runners.dataflow.native_io import iobase as native_iobase from apache_beam.utils import counters -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.worker import operation_specs from apache_beam.runners.worker import operations http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/runners/worker/sdk_worker_main.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py index 9c11068..f3f1e02 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py @@ -24,7 +24,7 @@ import sys import grpc from google.protobuf import text_format -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.worker.log_handler import FnApiLogRecordHandler from apache_beam.runners.worker.sdk_worker import SdkHarness http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/runners/worker/sdk_worker_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py index 93f60d3..c431bcd 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py @@ -29,7 +29,7 @@ import grpc from apache_beam.io.concat_source_test import RangeSource from apache_beam.io.iobase import SourceBundle -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.worker import data_plane from apache_beam.runners.worker import sdk_worker http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/transforms/core.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py index d7fa770..a137a13 100644 --- a/sdks/python/apache_beam/transforms/core.py +++ b/sdks/python/apache_beam/transforms/core.py @@ -27,7 +27,7 @@ from apache_beam import pvalue from apache_beam import typehints from apache_beam.coders import typecoders from apache_beam.internal import util -from apache_beam.portability.runners.api import beam_runner_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.transforms import ptransform from apache_beam.transforms.display import DisplayDataItem from apache_beam.transforms.display import HasDisplayData http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/transforms/ptransform.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py index 79fe3ad..6041353 100644 --- a/sdks/python/apache_beam/transforms/ptransform.py +++ b/sdks/python/apache_beam/transforms/ptransform.py @@ -430,7 +430,7 @@ class PTransform(WithTypeHints, HasDisplayData): cls._known_urns[urn] = parameter_type, constructor def to_runner_api(self, context): - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 urn, typed_param = self.to_runner_api_parameter(context) return beam_runner_api_pb2.FunctionSpec( urn=urn, http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/transforms/trigger.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py index 4151607..89c6ec5 100644 --- a/sdks/python/apache_beam/transforms/trigger.py +++ b/sdks/python/apache_beam/transforms/trigger.py @@ -33,7 +33,7 @@ from apache_beam.transforms.window import GlobalWindow from apache_beam.transforms.window import TimestampCombiner from apache_beam.transforms.window import WindowedValue from apache_beam.transforms.window import WindowFn -from apache_beam.portability.runners.api import beam_runner_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.utils.timestamp import MAX_TIMESTAMP from apache_beam.utils.timestamp import MIN_TIMESTAMP http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/transforms/window.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py index 08c7a2d..458fb74 100644 --- a/sdks/python/apache_beam/transforms/window.py +++ b/sdks/python/apache_beam/transforms/window.py @@ -55,8 +55,8 @@ from google.protobuf import duration_pb2 from google.protobuf import timestamp_pb2 from apache_beam.coders import coders -from apache_beam.portability.runners.api import beam_runner_api_pb2 -from apache_beam.portability.runners.api import standard_window_fns_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 +from apache_beam.portability.api import standard_window_fns_pb2 from apache_beam.transforms import timeutil from apache_beam.utils import proto_utils from apache_beam.utils import urns http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/apache_beam/utils/urns.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/urns.py b/sdks/python/apache_beam/utils/urns.py index b925bcc..e553eea 100644 --- a/sdks/python/apache_beam/utils/urns.py +++ b/sdks/python/apache_beam/utils/urns.py @@ -102,7 +102,7 @@ class RunnerApiFn(object): Prefer overriding self.to_runner_api_parameter. """ - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 urn, typed_param = self.to_runner_api_parameter(context) return beam_runner_api_pb2.SdkFunctionSpec( spec=beam_runner_api_pb2.FunctionSpec( http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/gen_protos.py ---------------------------------------------------------------------- diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py index a33c74b..a3d963d 100644 --- a/sdks/python/gen_protos.py +++ b/sdks/python/gen_protos.py @@ -35,7 +35,7 @@ BEAM_PROTO_PATHS = [ os.path.join('..', 'common', 'fn-api', 'src', 'main', 'proto') ] -PYTHON_OUTPUT_PATH = os.path.join('apache_beam', 'portability', 'runners', 'api') +PYTHON_OUTPUT_PATH = os.path.join('apache_beam', 'portability', 'api') def generate_proto_files(): http://git-wip-us.apache.org/repos/asf/beam/blob/7e4e51fd/sdks/python/run_pylint.sh ---------------------------------------------------------------------- diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index 7434516..2691be4 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -46,7 +46,7 @@ EXCLUDED_GENERATED_FILES=( "apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py" "apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py" "apache_beam/coders/proto2_coder_test_messages_pb2.py" -apache_beam/portability/runners/api/*pb2*.py +apache_beam/portability/api/*pb2*.py ) FILES_TO_IGNORE=""