This is an automated email from the ASF dual-hosted git repository. altay pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new c32acce Add common code infrastructure for element-wise snippets new 3124531 Merge pull request #8811 from davidcavazos/element-wise-snippets c32acce is described below commit c32accec7b6d254851d156facf03154198cf2e54 Author: David Cavazos <davido...@gmail.com> AuthorDate: Mon Jun 10 16:01:56 2019 -0700 Add common code infrastructure for element-wise snippets --- .../examples/snippets/transforms/__init__.py | 18 ++++++ .../snippets/transforms/element_wise/__init__.py | 18 ++++++ sdks/python/apache_beam/examples/snippets/util.py | 56 ++++++++++++++++++ .../apache_beam/examples/snippets/util_test.py | 67 ++++++++++++++++++++++ 4 files changed, 159 insertions(+) diff --git a/sdks/python/apache_beam/examples/snippets/transforms/__init__.py b/sdks/python/apache_beam/examples/snippets/transforms/__init__.py new file mode 100644 index 0000000..6569e3f --- /dev/null +++ b/sdks/python/apache_beam/examples/snippets/transforms/__init__.py @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import absolute_import diff --git a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/__init__.py b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/__init__.py new file mode 100644 index 0000000..6569e3f --- /dev/null +++ b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/__init__.py @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import absolute_import diff --git a/sdks/python/apache_beam/examples/snippets/util.py b/sdks/python/apache_beam/examples/snippets/util.py new file mode 100644 index 0000000..6e6e9e0 --- /dev/null +++ b/sdks/python/apache_beam/examples/snippets/util.py @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import absolute_import + +import argparse +import shlex +import subprocess as sp + + +def parse_example(argv=None): + """Parse the command line arguments and return it as a string function call. + + Examples: + python path/to/snippets.py function_name + python path/to/snippets.py function_name arg1 + python path/to/snippets.py function_name arg1 arg2 ... argN + """ + parser = argparse.ArgumentParser() + parser.add_argument('example', help='Name of the example to run.') + parser.add_argument('args', nargs=argparse.REMAINDER, + help='Arguments for example.') + args = parser.parse_args(argv) + + # Return the example as a string representing the Python function call. + example_args = ', '.join([repr(arg) for arg in args.args]) + return '{}({})'.format(args.example, example_args) + + +def run_shell_commands(commands, **kwargs): + """Runs a list of Notebook-like shell commands. + + Lines starting with `#` are ignored as comments. + Lines starting with `!` are run as commands. + Variables like `{variable}` are substituted with **kwargs. + """ + for cmd in commands: + cmd = cmd.strip().lstrip('!').format(**kwargs) + sp_cmd = shlex.split(cmd, comments=True, posix=True) + if sp_cmd: + sp.call(sp_cmd) + yield sp_cmd diff --git a/sdks/python/apache_beam/examples/snippets/util_test.py b/sdks/python/apache_beam/examples/snippets/util_test.py new file mode 100644 index 0000000..a23e916 --- /dev/null +++ b/sdks/python/apache_beam/examples/snippets/util_test.py @@ -0,0 +1,67 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import absolute_import + +import unittest + +from mock import patch + +from apache_beam.examples.snippets.util import * + + +class UtilTest(unittest.TestCase): + def test_parse_example_empty(self): + # python path/to/snippets.py + argv = [] + with self.assertRaises(SystemExit): + self.assertEqual(parse_example(argv), 'example()') + + def test_parse_example_no_arguments(self): + # python path/to/snippets.py example + argv = ['example'] + self.assertEqual(parse_example(argv), 'example()') + + def test_parse_example_one_argument(self): + # python path/to/snippets.py example A + argv = ['example', 'A'] + self.assertEqual(parse_example(argv), "example('A')") + + def test_parse_example_multiple_arguments(self): + # python path/to/snippets.py example A B "C's" + argv = ['example', 'A', 'B', "C's"] + self.assertEqual(parse_example(argv), "example('A', 'B', \"C's\")") + + @patch('subprocess.call', lambda cmd: None) + def test_run_shell_commands(self): + commands = [ + ' # this is a comment ', + ' ! echo this is a shell command ', + ' !echo {variable} ', + ' echo "quoted arguments work" # trailing comment ', + ] + actual = list(run_shell_commands(commands, variable='hello world')) + expected = [ + ['echo', 'this', 'is', 'a', 'shell', 'command'], + ['echo', 'hello', 'world'], + ['echo', 'quoted arguments work'], + ] + self.assertEqual(actual, expected) + + +if __name__ == '__main__': + unittest.main()