This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch tirx-standalone-root in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 904f3e0433443048997becc192ab34e0ffde5077 Author: tqchen <[email protected]> AuthorDate: Fri Oct 16 02:24:49 2026 +0000 [LINT] Add standalone TIRx lint configuration Move applicable lint hooks, formatters, and checks to the repository root. Preserve package settings while importing Ruff rules and keep CUDA native and Python generation checks independent of the TVM documentation tree. --- .clang-format | 8 + .github/workflows/lint.yml | 41 ++ .markdownlint-cli2.yaml | 19 + .pre-commit-config.yaml | 95 +++++ .yamllint.yaml | 32 ++ pyproject.toml | 49 +++ python/tvm_tirx/backend/cuda/launch/generate.py | 70 ---- tests/lint/check_asf_header.py | 486 ++++++++++++++++++++++++ tests/lint/check_file_type.py | 137 +++++++ 9 files changed, 867 insertions(+), 70 deletions(-) diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000000..b61ed538a8 --- /dev/null +++ b/.clang-format @@ -0,0 +1,8 @@ +# Run the following command to reformat a file: +# clang-format -i -style=Google <file> +# Or use clang-format-diff to only reformat the changed lines: +# https://clang.llvm.org/docs/ClangFormat.html +BasedOnStyle: Google +DerivePointerAlignment: false +ColumnLimit: 100 +PointerAlignment: Left diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..a9bff51f3b --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Lint + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: Lint-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/[email protected] + with: + fetch-depth: 0 + fetch-tags: true + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml new file mode 100644 index 0000000000..8d56f07c61 --- /dev/null +++ b/.markdownlint-cli2.yaml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +config: + MD013: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..3d0139548a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +default_install_hook_types: + - pre-commit +repos: + # ---------- Local hooks ---------- + - repo: local + hooks: + - id: cuda-launch-generated + name: check CUDA launch generated files + entry: python python/tvm_tirx/backend/cuda/launch/generate.py --check + language: python + additional_dependencies: [ruff==0.12.3, clang-format==20.1.8] + pass_filenames: false + files: ^(python/tvm_tirx/backend/cuda/launch/|src/backend/cuda/launch/) + - id: check-asf-header + name: check ASF Header + entry: python tests/lint/check_asf_header.py --check + language: python + language_version: python3 + pass_filenames: false + verbose: false + - id: check-file-type + name: check file types + entry: python tests/lint/check_file_type.py + language: python + language_version: python3 + pass_filenames: false + verbose: false + # ---------- General file checks ---------- + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: end-of-file-fixer + exclude: ^3rdparty/ + - id: mixed-line-ending + exclude: ^3rdparty/ + - id: requirements-txt-fixer + - id: trailing-whitespace + exclude: ^3rdparty/ + - id: check-yaml + - id: check-toml + + # ---------- YAML lint ---------- + - repo: https://github.com/adrienverge/yamllint + rev: v1.35.1 + hooks: + - id: yamllint + args: + - --config-file + - .yamllint.yaml + exclude: ^\.github/ + + # ---------- TOML format ---------- + - repo: https://github.com/ComPWA/taplo-pre-commit + rev: v0.9.3 + hooks: + - id: taplo-format + + # ---------- Python: ruff ---------- + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.3 + hooks: + - id: ruff-check + types_or: [python, pyi, jupyter] + args: [--fix] + - id: ruff-format + types_or: [python, pyi, jupyter] + + # ---------- C/C++: clang-format ---------- + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: "v20.1.8" + hooks: + - id: clang-format + types_or: [c, c++, cuda, objective-c++] + exclude: ^3rdparty/ diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000000..417d517367 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +extends: default + +rules: + document-start: disable + line-length: + max: 120 + level: warning + truthy: + allowed-values: + - "on" + - "off" + - "yes" + - "no" + - "true" + - "false" diff --git a/pyproject.toml b/pyproject.toml index 69fb8a3069..1288b4174d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -164,3 +164,52 @@ rm -rf "$DELOCATE_TMP_DIR" "$TMP_DIR" [tool.cibuildwheel.windows] before-build = 'python -m pip install "delvewheel>=1.12.0"' repair-wheel-command = 'delvewheel repair --analyze-existing --ignore-existing --exclude "*tvm_ffi*.dll" --exclude tvm_runtime_cuda.dll --exclude nvcuda.dll --exclude cudart64_13.dll --exclude nvrtc64_130_0.dll -w {dest_dir} {wheel}' + +[tool.ruff] +include = [ + "python/**/*.py", + "tests/**/*.py", + "docs/**/*.py", + "ci/**/*.py", + "conftest.py", +] +line-length = 100 +indent-width = 4 +target-version = "py310" +exclude = ["3rdparty", "build", "dist", ".venv", ".ruff_cache", "node_modules"] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors, https://docs.astral.sh/ruff/rules/#pycodestyle-e-w + "F", # pyflakes, https://docs.astral.sh/ruff/rules/#pyflakes-f + "I", # isort, https://docs.astral.sh/ruff/rules/#isort-i + "UP", # pyupgrade, https://docs.astral.sh/ruff/rules/#pyupgrade-up + "RUF", # ruff, https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf +] +ignore = [ + # pyupgrade (style preference or low-value) + "UP028", # yield-in-for-loop — sometimes clearer to keep + "UP030", # format-literals — minor + "UP031", # printf-string-formatting — style preference + # ruff (preview rules, not yet stable) + "RUF043", # pytest-raises-ambiguous-pattern + "RUF059", # unused-unpacked-variable +] +fixable = ["ALL"] +unfixable = [] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["E402", "F401", "F403", "F405"] +# TIRx parser and lowering tests intentionally retain unused IR bindings. +"tests/python/tirx/**/*.py" = ["F841"] + +[tool.ruff.lint.isort] +known-first-party = ["tvm_tirx"] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" +docstring-code-format = false +docstring-code-line-length = "dynamic" diff --git a/python/tvm_tirx/backend/cuda/launch/generate.py b/python/tvm_tirx/backend/cuda/launch/generate.py index 5e546a650b..e0caea02cf 100644 --- a/python/tvm_tirx/backend/cuda/launch/generate.py +++ b/python/tvm_tirx/backend/cuda/launch/generate.py @@ -218,75 +218,6 @@ def native_attributes(): return "\n".join(out) -def documentation(): - out = [ - ".. Licensed to the Apache Software Foundation (ASF) under one", - " or more contributor license agreements. See the NOTICE file", - " distributed with this work for additional information", - " regarding copyright ownership. The ASF licenses this file", - ' to you under the Apache License, Version 2.0 (the "License");', - " you may not use this file except in compliance with the License.", - " You may obtain a copy of the License at", - "", - " http://www.apache.org/licenses/LICENSE-2.0";, - "", - " Unless required by applicable law or agreed to in writing,", - ' software distributed under the License is distributed on an "AS IS" BASIS,', - " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.", - " See the License for the specific language governing permissions and", - " limitations under the License.", - "", - ".. Generated by launch/generate.py; edit table.py instead.", - "", - "CUDA kernel configuration reference", - "===================================", - "", - "Both CUDA module launches and exported ``cuda_host`` wrappers use this registry.", - "``LaunchConfig`` describes each launch; ``KernelAttributes`` describes compile-time", - "CUDA kernel attributes.", - "Optional launch attributes default to unspecified, preserving CUDA's inherited defaults.", - "", - ] - for name, fields in [ - ("LaunchConfig", table.LAUNCH_FIELDS), - ("KernelAttributes", table.KERNEL_ATTR_FIELDS), - *table.COMPOSITES.items(), - ]: - out += [ - name, - "-" * len(name), - "", - ".. list-table::", - " :header-rows: 1", - "", - " * - Field", - " - Type", - " - Default", - " - Description", - ] - for field in fields: - doc = field.doc - if field.enums: - doc += " Values: " + ", ".join(f"``{key}``" for key, _ in field.enums) + "." - if field.driver_id: - doc += ( - f" Encoder requires CUDA {field.version // 1000}.{field.version % 1000 // 10}+." - ) - out += [ - f" * - ``{field.name}``", - f" - {field.kind}", - f" - ``{field.default}``", - f" - {doc}", - ] - out += [""] - out += [ - "Stream-only synchronization policy and graph-only device-updatable nodes are", - "outside this kernel-launch API. CUDA checks device-specific support and resource limits.", - "", - ] - return "\n".join(out) - - def format_output(path, content): """Use the same pinned formatters as pre-commit, without importing TVM.""" if path.suffix == ".py": @@ -318,7 +249,6 @@ def main(): DIRECTORY / "_generated.py": python_api(), ROOT / "src/backend/cuda/launch/launch_fields.h": native_fields(), ROOT / "src/backend/cuda/launch/launch_attributes.h": native_attributes(), - ROOT.parent / "docs/tirx/api/cuda_launch.rst": documentation(), } for path in list(outputs): outputs[path] = format_output(path, outputs[path]) diff --git a/tests/lint/check_asf_header.py b/tests/lint/check_asf_header.py new file mode 100644 index 0000000000..86301e2502 --- /dev/null +++ b/tests/lint/check_asf_header.py @@ -0,0 +1,486 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ruff: noqa: E741 +"""Helper tool to check and fix ASF headers in source files tracked by git. + +Ported from tvm-ffi. Replaces the previous RAT-based check_asf_header.sh +(which required Java) with a self-contained Python implementation. +""" + +from __future__ import annotations + +import argparse +import fnmatch +import subprocess +import sys +from pathlib import Path + +header_cstyle = """ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +""".strip() + +header_mdstyle = """ +<!--- Licensed to the Apache Software Foundation (ASF) under one --> +<!--- or more contributor license agreements. See the NOTICE file --> +<!--- distributed with this work for additional information --> +<!--- regarding copyright ownership. The ASF licenses this file --> +<!--- to you under the Apache License, Version 2.0 (the --> +<!--- "License"); you may not use this file except in compliance --> +<!--- with the License. You may obtain a copy of the License at --> + +<!--- http://www.apache.org/licenses/LICENSE-2.0 --> + +<!--- Unless required by applicable law or agreed to in writing, --> +<!--- software distributed under the License is distributed on an --> +<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --> +<!--- KIND, either express or implied. See the License for the --> +<!--- specific language governing permissions and limitations --> +<!--- under the License. --> +""".strip() + +header_pystyle = """ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""".strip() + +header_rststyle = """ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +""".strip() + +header_groovystyle = """ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +""".strip() + +header_cmdstyle = """ +:: Licensed to the Apache Software Foundation (ASF) under one +:: or more contributor license agreements. See the NOTICE file +:: distributed with this work for additional information +:: regarding copyright ownership. The ASF licenses this file +:: to you under the Apache License, Version 2.0 (the +:: "License"); you may not use this file except in compliance +:: with the License. You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, +:: software distributed under the License is distributed on an +:: "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +:: KIND, either express or implied. See the License for the +:: specific language governing permissions and limitations +:: under the License. +""".strip() + +FMT_MAP = { + "sh": header_pystyle, + "cc": header_cstyle, + "c": header_cstyle, + "cu": header_cstyle, + "cuh": header_cstyle, + "mm": header_cstyle, + "m": header_cstyle, + "go": header_cstyle, + "java": header_cstyle, + "h": header_cstyle, + "pyi": header_pystyle, + "py": header_pystyle, + "pyx": header_pystyle, + "toml": header_pystyle, + "yml": header_pystyle, + "yaml": header_pystyle, + "rs": header_cstyle, + "md": header_mdstyle, + "cmake": header_pystyle, + "mk": header_pystyle, + "rst": header_rststyle, + "gradle": header_groovystyle, + "groovy": header_groovystyle, + "tcl": header_pystyle, + "xml": header_mdstyle, + "storyboard": header_mdstyle, + "pbxproj": header_cstyle, + "plist": header_mdstyle, + "xcworkspacedata": header_mdstyle, + "html": header_mdstyle, + "bat": header_cmdstyle, +} + +# Files and patterns to skip during header checking. +# Files and patterns to skip (3rdparty, generated files, etc.). +SKIP_LIST: list[str] = [ + "3rdparty/*", + ".github/*", + "*.json", + "*.txt", + "*.svg", + "*.lst", + "*.lds", + "*.in", + "*.diff", + "*.edl", + "*.md5", + "*.csv", + "*.log", + "*.interp", + "*.tokens", + "*.ipynb", + "*.conf", + "*.ini", + "*.lock", + "*.properties", + "*.j2", +] + + +def should_skip_file(filepath: str) -> bool: + """Check if file should be skipped based on SKIP_LIST.""" + for pattern in SKIP_LIST: + if fnmatch.fnmatch(filepath, pattern): + return True + return False + + +def get_git_files() -> list[str] | None: + """Get list of files tracked by git (excluding files staged for deletion).""" + try: + result = subprocess.run( + ["git", "ls-files"], check=False, capture_output=True, text=True, cwd=Path.cwd() + ) + if result.returncode != 0: + print("Error: Could not get git files. Make sure you're in a git repository.") + print("Git command failed:", result.stderr.strip()) + return None + all_files = {line.strip() for line in result.stdout.split("\n") if line.strip()} + # Exclude files staged for deletion so the header check does not + # report errors for files that are intentionally being removed. + deleted_result = subprocess.run( + ["git", "ls-files", "--deleted"], + check=False, + capture_output=True, + text=True, + cwd=Path.cwd(), + ) + if deleted_result.returncode == 0: + deleted = {line.strip() for line in deleted_result.stdout.split("\n") if line.strip()} + all_files -= deleted + elif deleted_result.stderr: + print(f"Warning: 'git ls-files --deleted' failed: {deleted_result.stderr.strip()}") + return sorted(all_files) + except FileNotFoundError: + print("Error: Git not found. This tool requires git to be installed.") + return None + + +def copyright_line(line: str) -> bool: + # Following two items are intentionally break apart + # so that the copyright detector won"t detect the file itself. + if line.find("Copyright " + "(c)") != -1: + return True + # break pattern into two lines to avoid false-negative check + spattern1 = "Copyright" + if line.find(spattern1) != -1 and line.find("by") != -1: + return True + return False + + +def check_header(fname: str, header: str) -> bool: + """Check header status of file without modifying it.""" + if not Path(fname).exists(): + print(f"ERROR: Cannot find {fname}") + return False + + lines = Path(fname).open().readlines() + + has_asf_header = False + has_copyright = False + + for i, l in enumerate(lines): + if l.find("Licensed to the Apache Software Foundation") != -1: + has_asf_header = True + elif copyright_line(l): + has_copyright = True + + if has_asf_header and not has_copyright: + return True # File is good + + if not has_asf_header: + print(f"ERROR: Missing ASF header in {fname}") + print("Run `python tests/lint/check_asf_header.py --fix` to add the header") + return False + + if has_copyright: + print(f"ERROR: Has copyright line that should be removed in {fname}") + return False + + return True + + +def collect_files() -> list[str] | None: + """Collect all files that need header checking from git.""" + files = [] + + # Get files from git (required) + git_files = get_git_files() + + if git_files is None: + # Git is required, so exit if we can't get files + return None + + # Filter git files based on supported types and skip list + for git_file in git_files: + # Check if file should be skipped + if should_skip_file(git_file): + continue + + # Check if this file type is supported + suffix = git_file.split(".")[-1] if "." in git_file else "" + basename = Path(git_file).name + + if ( + suffix in FMT_MAP + or basename == "gradle.properties" + or (suffix == "" and basename in ["CMakeLists", "Makefile"]) + ): + files.append(git_file) + + return files + + +def add_header(fname: str, header: str) -> None: + """Add header to file.""" + if not Path(fname).exists(): + print(f"Cannot find {fname} ...") + return + + lines = Path(fname).open().readlines() + + has_asf_header = False + has_copyright = False + + for i, l in enumerate(lines): + if l.find("Licensed to the Apache Software Foundation") != -1: + has_asf_header = True + elif copyright_line(l): + has_copyright = True + lines[i] = "" + + if has_asf_header and not has_copyright: + return + + with Path(fname).open("w") as outfile: + skipline = False + if not lines: + skipline = False # File is empty + elif lines[0][:2] == "#!": + skipline = True + elif lines[0][:2] == "<?": + skipline = True + elif lines[0].startswith("<html>"): + skipline = True + elif lines[0].startswith("// !$"): + skipline = True + + if skipline: + outfile.write(lines[0]) + if not has_asf_header: + outfile.write(header + "\n\n") + outfile.write("".join(lines[1:])) + else: + if not has_asf_header: + outfile.write(header + "\n\n") + outfile.write("".join(lines)) + if not has_asf_header: + print(f"Add header to {fname}") + if has_copyright: + print(f"Removed copyright line from {fname}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check and fix ASF headers in source files tracked by git", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +This tool processes all files tracked by git (using 'git ls-files') and automatically +skips files matching patterns in SKIP_LIST (build artifacts, third-party files, etc.). + +One of --check, --fix, or --dry-run must be specified. + +Examples: + # Show all files that would be processed (dry run) + python check_asf_header.py --dry-run + + # Check all git-tracked files (report errors) + python check_asf_header.py --check + + # Fix all git-tracked files (add/update headers) + python check_asf_header.py --fix + """, + ) + + parser.add_argument( + "--check", + action="store_true", + help="Check mode: report errors without modifying files", + ) + + parser.add_argument( + "--fix", + action="store_true", + help="Fix mode: fix files with missing or incorrect headers (default)", + ) + + parser.add_argument( + "--dry-run", + action="store_true", + help="Dry run: show files that would be processed without doing anything", + ) + + args = parser.parse_args() + + # Validate arguments + if args.check and args.fix: + print("Error: Cannot specify both --check and --fix") + return 1 + + if not args.check and not args.fix and not args.dry_run: + print("Error: Must specify one of --check, --fix, or --dry-run") + return 1 + + # Always use git-based approach + files = collect_files() + + if files is None: + return 1 # Error already printed by get_git_files() + + if not files: + print("No files found to process") + return 0 + + # Handle dry-run mode + if args.dry_run: + print(f"Would process {len(files)} files:") + for fname in sorted(files): + print(f" {fname}") + return 0 + + # Determine mode + check_only = args.check + + error_count = 0 + processed_count = 0 + + for fname in files: + processed_count += 1 + suffix = fname.split(".")[-1] if "." in fname else "" + basename = Path(fname).name + + # Determine header type + if suffix in FMT_MAP: + header = FMT_MAP[suffix] + elif basename == "gradle.properties": + header = FMT_MAP["h"] + elif suffix == "" and basename in ["CMakeLists", "Makefile"]: + header = FMT_MAP["cmake"] if basename == "CMakeLists" else FMT_MAP["mk"] + else: + if check_only: + print(f"ERROR: Cannot handle file type for {fname}") + error_count += 1 + else: + print(f"Cannot handle {fname} ...") + continue + + if check_only: + # Check mode + if not check_header(fname, header): + error_count += 1 + else: + # Fix mode + add_header(fname, header) + + if check_only: + if error_count > 0: + print(f"\nFound {error_count} errors in {processed_count} files") + return 1 + else: + print(f"\nAll {processed_count} files are compliant") + return 0 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/lint/check_file_type.py b/tests/lint/check_file_type.py new file mode 100644 index 0000000000..d4fe94b207 --- /dev/null +++ b/tests/lint/check_file_type.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Helper tool to check file types that are allowed to checkin.""" + +import subprocess +import sys +from pathlib import Path + +# List of file types we allow +ALLOW_EXTENSION = { + # source code + "cc", + "c", + "h", + "S", + "cu", + "cuh", + "m", + "mm", + "groovy", + "sh", + "bat", + "py", + "pyi", + # configurations + "cfg", + "mk", + "in", + "cmake", + "xml", + "toml", + "yml", + "yaml", + "json", + "cfg", + "ini", + # docs + "txt", + "md", + "rst", + "css", + "html", + "ipynb", + # Jinja2 templates + "j2", + # images + "png", + # misc + "properties", + "template", +} + +# List of file names allowed +ALLOW_FILE_NAME = { + ".gitignore", + ".gitattributes", + ".gitmodules", + ".clang-format", + "README", + "Makefile", + "Doxyfile", +} + +# List of specific files allowed in relpath to <proj_root> +ALLOW_SPECIFIC_FILE = {"LICENSE", "NOTICE", "KEYS"} + + +def filename_allowed(name: str) -> bool: + """Check if name is allowed by the current policy. + + Parameters + ---------- + name : str + Input name + + Returns + ------- + allowed : bool + Whether the filename is allowed. + """ + arr = name.rsplit(".", 1) + if arr[-1] in ALLOW_EXTENSION: + return True + + basename = Path(name).name + if basename in ALLOW_FILE_NAME: + return True + + if name.startswith("3rdparty/"): + return True + + if name in ALLOW_SPECIFIC_FILE: + return True + + return False + + +def main() -> None: + cmd = ["git", "ls-files"] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + (out, _) = proc.communicate() + res = out.decode("utf-8") + assert proc.returncode == 0, f"{' '.join(cmd)} errored: {res}" + error_list = [f for f in res.split() if not filename_allowed(f)] + + if error_list: + report = "------File type check report----\n" + report += "\n".join(error_list) + report += f"\nFound {len(error_list)} files that are not allowed\n" + report += "We do not check in binary files into the repo.\n" + report += ( + "If necessary, please discuss with committers and " + "modify tests/lint/check_file_type.py to enable the file you need.\n" + ) + sys.stderr.write(report) + sys.stderr.flush() + sys.exit(-1) + + print("check_file_type.py: all checks passed..") + + +if __name__ == "__main__": + main()
