This is an automated email from the ASF dual-hosted git repository.
lostluck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new bdd29bf45f8 Add script to fetch SCIO examples (#26891)
bdd29bf45f8 is described below
commit bdd29bf45f818ce0c20c111c9787d5051c1b792b
Author: Timur Sultanov <timur.sulta...@akvelon.com>
AuthorDate: Wed May 31 22:37:52 2023 +0400
Add script to fetch SCIO examples (#26891)
---
playground/infrastructure/fetch_scala_examples.py | 245 ++++++++++++++++++++++
playground/infrastructure/requirements.txt | 2 +
2 files changed, 247 insertions(+)
diff --git a/playground/infrastructure/fetch_scala_examples.py
b/playground/infrastructure/fetch_scala_examples.py
new file mode 100644
index 00000000000..6a8682c8a10
--- /dev/null
+++ b/playground/infrastructure/fetch_scala_examples.py
@@ -0,0 +1,245 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from dataclasses import dataclass
+from typing import List, Tuple
+
+import argparse
+import requests
+import re
+
+from models import ComplexityEnum
+
+SCIO_REPOSITORY = "https://raw.githubusercontent.com/spotify/scio/";
+SCIO_BRANCH = "main"
+
+
+@dataclass
+class ScioExampleTag:
+ filepath: str
+ name: str
+ description: str
+ multifile: bool
+ pipeline_options: str
+ default_example: bool
+ context_line: int
+ categories: List[str]
+ complexity: ComplexityEnum
+ tags: List[str]
+
+
+SCIO_EXAMPLES: List[ScioExampleTag] = [
+ ScioExampleTag(
+
filepath="scio-examples/src/main/scala/com/spotify/scio/examples/DebuggingWordCount.scala",
+ name="DebuggingWordCount",
+ description="Word Count Example with Assertions.",
+ multifile=False,
+ pipeline_options="--output output.txt",
+ default_example=False,
+ context_line=1,
+ categories=["Debugging", "Filtering", "Options", "Quickstart"],
+ complexity=ComplexityEnum.MEDIUM,
+ tags=["Example"]
+ ),
+ ScioExampleTag(
+
filepath="scio-examples/src/main/scala/com/spotify/scio/examples/extra/MetricsExample.scala",
+ name="MetricsExample",
+ description="Metrics example.",
+ multifile=False,
+ pipeline_options="",
+ default_example=False,
+ context_line=1,
+ categories=["Combiners", "Filtering", "IO", "Core Transforms",
"Quickstart"],
+ complexity=ComplexityEnum.MEDIUM,
+ tags=["Example"]
+ ),
+ ScioExampleTag(
+
filepath="scio-examples/src/main/scala/com/spotify/scio/examples/MinimalWordCount.scala",
+ name="MinimalWordCount",
+ description="An example that counts words in Shakespeare's works.",
+ multifile=False,
+ pipeline_options="--output output.txt",
+ default_example=True,
+ context_line=1,
+ categories=["Combiners", "Filtering", "IO", "Core Transforms",
"Quickstart"],
+ complexity=ComplexityEnum.BASIC,
+ tags=["Example"]
+ ),
+ ScioExampleTag(
+
filepath="scio-examples/src/main/scala/com/spotify/scio/examples/extra/SafeFlatMapExample.scala",
+ name="SafeFlatMapExample",
+ description="SafeFlatMap usage",
+ multifile=False,
+ pipeline_options="--output output.txt",
+ default_example=False,
+ context_line=1,
+ categories=["Combiners", "Filtering", "IO", "Core Transforms",
"Quickstart"],
+ complexity=ComplexityEnum.MEDIUM,
+ tags=["Example"]
+ ),
+ ScioExampleTag(
+
filepath="scio-examples/src/main/scala/com/spotify/scio/examples/extra/StatefulExample.scala",
+ name="StatefulExample",
+ description="Stateful Processing.",
+ multifile=False,
+ pipeline_options="",
+ default_example=False,
+ context_line=1,
+ categories=["Combiners", "Filtering", "IO", "Core Transforms",
"Quickstart"],
+ complexity=ComplexityEnum.MEDIUM,
+ tags=["Example"]
+ ),
+ ScioExampleTag(
+
filepath="scio-examples/src/main/scala/com/spotify/scio/examples/complete/TfIdf.scala",
+ name="TfIdf",
+ description="Compute TF-IDF from a Text Corpus.",
+ multifile=False,
+ pipeline_options="--output output.txt",
+ default_example=False,
+ context_line=1,
+ categories=["Combiners", "Filtering", "IO", "Core Transforms",
"Quickstart"],
+ complexity=ComplexityEnum.MEDIUM,
+ tags=["Example"]
+ ),
+ ScioExampleTag(
+
filepath="scio-examples/src/main/scala/com/spotify/scio/examples/extra/TsvExample.scala",
+ name="TsvExampleWrite",
+ description="Reading and writing tsv data.",
+ multifile=False,
+ pipeline_options="--output output.txt",
+ default_example=False,
+ context_line=1,
+ categories=["Combiners", "Filtering", "IO", "Core Transforms",
"Quickstart"],
+ complexity=ComplexityEnum.MEDIUM,
+ tags=["Example"]
+ ),
+ ScioExampleTag(
+
filepath="scio-examples/src/main/scala/com/spotify/scio/examples/WordCount.scala",
+ name="WordCount",
+ description="An example that counts words in Shakespeare's works.",
+ multifile=False,
+ pipeline_options="--output output.txt",
+ default_example=False,
+ context_line=1,
+ categories=["Combiners", "Options", "Quickstart"],
+ complexity=ComplexityEnum.MEDIUM,
+ tags=["Example"]
+ ),
+ ScioExampleTag(
+
filepath="scio-examples/src/main/scala/com/spotify/scio/examples/extra/WordCountScioIO.scala",
+ name="WordCountScioIO",
+ description="Word Count Example with Metrics and ScioIO read/write.",
+ multifile=False,
+ pipeline_options="--output output.txt",
+ default_example=False,
+ context_line=1,
+ categories=["Combiners", "Filtering", "IO", "Core Transforms",
"Quickstart"],
+ complexity=ComplexityEnum.MEDIUM,
+ tags=["Example"]
+ ),
+ ScioExampleTag(
+
filepath="scio-examples/src/main/scala/com/spotify/scio/examples/extra/WriteDynamicExample.scala",
+ name="WriteDynamicExample",
+ description="Demonstrates saveAsDynamic* methods.",
+ multifile=False,
+ pipeline_options="--output output.txt",
+ default_example=False,
+ context_line=1,
+ categories=["Combiners", "Filtering", "IO", "Core Transforms",
"Quickstart"],
+ complexity=ComplexityEnum.MEDIUM,
+ tags=["Example"]
+ ),
+]
+
+
+def fetch_scala_examples() -> Tuple[ScioExampleTag, str]:
+ """Fetch all Scala examples from the Scio repository."""
+ urls = [(example, SCIO_REPOSITORY + "/" + SCIO_BRANCH + "/" +
example.filepath) for example in SCIO_EXAMPLES]
+ for example, url in urls:
+ result = requests.get(url)
+ if result.status_code != 200:
+ print(f"Failed to fetch {url} with status code
{result.status_code}, skipping")
+ continue
+ content = result.text
+ yield example, content
+
+
+def serialize_tag_to_yaml(tag: ScioExampleTag) -> str:
+ """Serialize a Tag to YAML."""
+
+ yaml = f"""beam-playground:
+ name: "{tag.name}"
+ description: "{tag.description}"
+ multifile: {tag.multifile}
+ pipeline_options: "{tag.pipeline_options}"
+ default_example: {tag.default_example}
+ context_line: {tag.context_line}
+ categories:
+"""
+ for category in tag.categories:
+ yaml += f" - \"{category}\"\n"
+ yaml += f" complexity: {tag.complexity}\n"
+ yaml += f" tags:\n"
+ for t in tag.tags:
+ yaml += f" - \"{t}\"\n"
+ return yaml
+
+
+def insert_tag_into_source(tag_yaml: str, source: str) -> str:
+ """Insert the tag YAML into the source code."""
+
+ lines = source.split("\n")
+ package_line = 0
+ for i, line in enumerate(lines):
+ if line.startswith("package"):
+ package_line = i
+ break
+
+ object_line = 0
+ for i, line in enumerate(lines):
+ if line.startswith("object"):
+ object_line = i
+ break
+
+ tag_lines_number = tag_yaml.count("\n")
+ tag_yaml = re.sub(r"context_line: \d+", f"context_line: {object_line +
tag_lines_number + 3}", tag_yaml)
+ tag_yaml = "// " + tag_yaml.replace("\n", "\n// ")
+ tag_yaml = "\n" + tag_yaml
+
+ # Insert the tag YAML before the package definition
+ lines.insert(package_line, tag_yaml)
+
+ return "\n".join(lines)
+
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument("--output-dir", dest="output_dir", help="Output
directory", required=True)
+
+if __name__ == "__main__":
+ args = argparser.parse_args()
+
+ if not os.path.exists(args.output_dir):
+ os.makedirs(args.output_dir)
+
+ for tag, source in fetch_scala_examples():
+ tag_yaml = serialize_tag_to_yaml(tag)
+ source = insert_tag_into_source(tag_yaml, source)
+
+ output_path = os.path.join(args.output_dir, tag.name + ".scala")
+
+ with open(output_path, "w") as f:
+ f.write(source)
+
+ print(f"Written {output_path}")
diff --git a/playground/infrastructure/requirements.txt
b/playground/infrastructure/requirements.txt
index 936b077129d..a5023df8d0a 100644
--- a/playground/infrastructure/requirements.txt
+++ b/playground/infrastructure/requirements.txt
@@ -26,3 +26,5 @@ pydantic==1.10.2
grpcio-tools==1.51.1
protobuf==4.21.12
google-cloud-datastore==2.11.0
+
+requests~=2.28.2
