792beca2c5
Sanitized open-source starter: renamed to daedalus, Norse-free of internal references, AI attribution stripped, skill merged into AGENTS.md, CI converted to Gitea Actions. No registered scripts yet.
151 lines
5.1 KiB
Python
151 lines
5.1 KiB
Python
"""File checks shared by autotest.py and the sanity job.
|
|
|
|
Syntax, lint, and format checks; each returns an error count.
|
|
"""
|
|
import ast
|
|
import json
|
|
import logging
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
|
|
import pycodestyle
|
|
import yaml
|
|
from pymarkdown.api import PyMarkdownApi
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
|
|
SKIP_DIRS = frozenset({
|
|
".git", ".venv", "venv", ".cache", ".pytest_cache", ".ruff_cache",
|
|
"__pycache__", "htmlcov", "node_modules", "build", "dist",
|
|
})
|
|
SHELL_SUFFIXES = (".sh", ".bash")
|
|
CRLF_SUFFIXES = (".bat",)
|
|
SHELL_SHEBANG = re.compile(rb"^#!.*\b(ba|da|k|z)?sh\b")
|
|
YAML_SUFFIXES = (".yaml", ".yml")
|
|
PYCODESTYLE_IGNORE = ["E501", "E121", "E123", "E126", "E226", "E24", "E704", "W503", "W504"]
|
|
MARKDOWN_IGNORE = frozenset({"md013", "md033"})
|
|
MAX_FILE_SIZE = 2_000_000
|
|
PRIVATE_KEY = re.compile(rb"-----BEGIN [A-Z ]*PRIVATE KEY-----")
|
|
|
|
|
|
def iter_files(root: pathlib.Path, suffixes: tuple[str, ...]) -> list[pathlib.Path]:
|
|
"""List files under root with a suffix, skipping SKIP_DIRS."""
|
|
found = []
|
|
for dirpath, dirnames, filenames in os.walk(root):
|
|
dirnames[:] = [name for name in dirnames if name not in SKIP_DIRS]
|
|
found.extend(pathlib.Path(dirpath, name) for name in filenames if pathlib.Path(name).suffix in suffixes)
|
|
return sorted(found)
|
|
|
|
|
|
def is_shell_script(path: pathlib.Path) -> bool:
|
|
"""A *.sh / *.bash file, or a suffixless file with a shell shebang."""
|
|
if path.suffix in SHELL_SUFFIXES:
|
|
return True
|
|
if path.suffix:
|
|
return False
|
|
with path.open("rb") as stream:
|
|
return SHELL_SHEBANG.match(stream.readline()) is not None
|
|
|
|
|
|
def check_python_syntax(path: pathlib.Path) -> int:
|
|
"""Return 1 when the Python file does not parse."""
|
|
try:
|
|
ast.parse(path.read_bytes(), filename=str(path))
|
|
except (SyntaxError, ValueError) as error:
|
|
logger.error(" [FAIL] python syntax %s: %s", path, error)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
def check_pycodestyle(paths: list[pathlib.Path]) -> int:
|
|
"""Return the number of PEP 8 violations in paths."""
|
|
style_guide = pycodestyle.StyleGuide(ignore=PYCODESTYLE_IGNORE, show_source=True)
|
|
return style_guide.check_files([str(path) for path in paths]).total_errors
|
|
|
|
|
|
def check_shell(path: pathlib.Path) -> int:
|
|
"""Return 1 when shellcheck reports a problem or is missing."""
|
|
shellcheck = shutil.which("shellcheck")
|
|
if shellcheck is None:
|
|
logger.error(" [FAIL] shellcheck not found; run: pip install -r requirements.txt")
|
|
return 1
|
|
result = subprocess.run(
|
|
[shellcheck, "--external-sources", str(path)],
|
|
capture_output=True, text=True, check=False,
|
|
)
|
|
if result.returncode != 0:
|
|
logger.error(" [FAIL] shellcheck %s\n%s%s", path, result.stdout, result.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
def check_json(path: pathlib.Path) -> int:
|
|
"""Return 1 when the file is not valid UTF-8 JSON."""
|
|
try:
|
|
json.loads(path.read_text(encoding="utf-8"))
|
|
except (json.JSONDecodeError, UnicodeDecodeError) as error:
|
|
logger.error(" [FAIL] json %s: %s", path, error)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
def check_yaml(path: pathlib.Path) -> int:
|
|
"""Return 1 when any YAML document in the file does not load."""
|
|
try:
|
|
list(yaml.safe_load_all(path.read_text(encoding="utf-8")))
|
|
except (yaml.YAMLError, UnicodeDecodeError) as error:
|
|
logger.error(" [FAIL] yaml %s: %s", path, error)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
def check_markdown(paths: list[pathlib.Path]) -> int:
|
|
"""Markdownlint failure count, minus MARKDOWN_IGNORE; YAML front matter allowed."""
|
|
errors = 0
|
|
for path in paths:
|
|
api = PyMarkdownApi().enable_extension_by_identifier("front-matter")
|
|
for failure in api.scan_path(str(path)).scan_failures:
|
|
if failure.rule_id.lower() in MARKDOWN_IGNORE:
|
|
continue
|
|
errors += 1
|
|
logger.error(" [FAIL] markdown %s:%s [%s] %s", path, failure.line_number,
|
|
failure.rule_id, failure.rule_description)
|
|
return errors
|
|
|
|
|
|
def check_line_ending(path: pathlib.Path) -> int:
|
|
"""Text files must be LF, CRLF_SUFFIXES files CRLF; binary files are skipped."""
|
|
data = path.read_bytes()
|
|
if b"\0" in data:
|
|
return 0
|
|
if path.suffix.lower() in CRLF_SUFFIXES:
|
|
if b"\n" not in data.replace(b"\r\n", b""):
|
|
return 0
|
|
logger.error(" [FAIL] %s needs CRLF line endings (cmd.exe)", path)
|
|
return 1
|
|
if b"\r" not in data:
|
|
return 0
|
|
logger.error(" [FAIL] CRLF line endings: %s", path)
|
|
return 1
|
|
|
|
|
|
def check_private_key(path: pathlib.Path) -> int:
|
|
"""Return 1 when the file contains a PEM or OpenSSH private key."""
|
|
if PRIVATE_KEY.search(path.read_bytes()) is None:
|
|
return 0
|
|
logger.error(" [FAIL] private key must not be committed: %s", path)
|
|
return 1
|
|
|
|
|
|
def check_file_size(path: pathlib.Path) -> int:
|
|
"""Return 1 when the file is larger than MAX_FILE_SIZE bytes."""
|
|
size = path.stat().st_size
|
|
if size <= MAX_FILE_SIZE:
|
|
return 0
|
|
logger.error(" [FAIL] %s is %d bytes (limit %d)", path, size, MAX_FILE_SIZE)
|
|
return 1
|