Files
phronesis/autotest.py
T
nvaldez b0f07593b5 phronesis v0.3.0: port the four known skills from vibe-wiki - five bundled total
- fiona 2.2.0 (productivity) with its support files (llm-wiki-mod.md, markdown_rules.md)
- memory 1.0.1 (productivity) with MEMORIES.md template; CRLF converted to LF
- sectionize-markdown 1.2.0 (documentation); category added, minor bump
- specification-writer 1.1.0 (documentation); category added, minor bump
- autotest markdown check: md025 front_matter_title set never-matching, so a
  front-matter 'title' key is not an implicit second h1 (SPEC item 9.6)
- README skills table lists all five; all 7 autotest checks green

Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
2026-11-19 10:20:04 +01:00

278 lines
10 KiB
Python

"""Run every phronesis check: syntax, lint, frontmatter, and markdown.
Single entry point for local runs and CI.
Usage:
.venv/bin/python autotest.py # every check
.venv/bin/python autotest.py frontmatter # only the named checks
"""
import argparse
import ast
import concurrent.futures
import functools
import logging
import os
import pathlib
import re
import shutil
import subprocess
import sys
import time
import pycodestyle
import yaml
from pymarkdown.api import PyMarkdownApi
logger = logging.getLogger(__name__)
REPO_ROOT = pathlib.Path(__file__).resolve().parent
SKILL_ROOT = REPO_ROOT / "skill"
SKIP_DIRS = frozenset({
".git", ".venv", "venv", ".cache", ".pytest_cache", ".ruff_cache",
"__pycache__", "htmlcov", "node_modules", "build", "dist",
})
SHELL_SUFFIXES = (".sh", ".bash")
CRLF_SUFFIXES = (".bat",)
SHELL_SHEBANG = re.compile(rb"^#!.*\b(ba|da|k|z)?sh\b")
VERSION = re.compile(r"^\d+\.\d+\.\d+$")
CATEGORY = re.compile(r"^[a-z][a-z0-9-]*$")
PRIVATE_KEY = re.compile(rb"-----BEGIN [A-Z ]*PRIVATE KEY-----")
FRONTMATTER_KEYS = ("name", "version", "description", "category")
PYCODESTYLE_IGNORE = ["E501", "E121", "E123", "E126", "E226", "E24", "E704", "W503", "W504"]
MARKDOWN_IGNORE = frozenset({"md013", "md033"})
MAX_FILE_SIZE = 2_000_000
MIN_COVERAGE_PERCENT = 75
def iter_files(root: pathlib.Path, suffixes: tuple[str, ...]) -> list[pathlib.Path]:
"""List files under root with a suffix, skipping SKIP_DIRS."""
found = []
for dirpath, dirnames, filenames in os.walk(root):
dirnames[:] = [name for name in dirnames if name not in SKIP_DIRS]
found.extend(pathlib.Path(dirpath, name) for name in filenames
if pathlib.Path(name).suffix in suffixes)
return sorted(found)
def is_shell_script(path: pathlib.Path) -> bool:
"""A *.sh / *.bash file, or a suffixless file with a shell shebang."""
if path.suffix in SHELL_SUFFIXES:
return True
if path.suffix:
return False
with path.open("rb") as stream:
return SHELL_SHEBANG.match(stream.readline()) is not None
def check_python_syntax(path: pathlib.Path) -> int:
"""Return 1 when the Python file does not parse."""
try:
ast.parse(path.read_bytes(), filename=str(path))
except (SyntaxError, ValueError) as error:
logger.error(" [FAIL] python syntax %s: %s", path, error)
return 1
return 0
def check_pycodestyle(paths: list[pathlib.Path]) -> int:
"""Return the number of PEP 8 violations in paths."""
style_guide = pycodestyle.StyleGuide(ignore=PYCODESTYLE_IGNORE, show_source=True)
return style_guide.check_files([str(path) for path in paths]).total_errors
def check_shell(path: pathlib.Path) -> int:
"""Return 1 when shellcheck reports a problem or is missing."""
shellcheck = shutil.which("shellcheck")
if shellcheck is None:
logger.error(" [FAIL] shellcheck not found; run: pip install -r requirements.txt")
return 1
result = subprocess.run(
[shellcheck, "--external-sources", str(path)],
capture_output=True, text=True, check=False,
)
if result.returncode != 0:
logger.error(" [FAIL] shellcheck %s\n%s%s", path, result.stdout, result.stderr)
return 1
return 0
def parse_frontmatter(path: pathlib.Path) -> dict | None:
"""Return the YAML front matter of a markdown file, or None."""
lines = path.read_text(encoding="utf-8").splitlines()
if not lines or lines[0].strip() != "---":
return None
try:
end = lines[1:].index("---") + 1
data = yaml.safe_load("\n".join(lines[1:end]))
except ValueError:
return None
except yaml.YAMLError:
return None
return data if isinstance(data, dict) else None
def check_frontmatter(path: pathlib.Path) -> int:
"""Check skill/<name>/SKILL.md front matter keys, name, and version."""
errors = 0
data = parse_frontmatter(path)
if data is None:
logger.error(" [FAIL] frontmatter %s: no '---' fenced YAML header", path)
return 1
for key in FRONTMATTER_KEYS:
if not data.get(key):
logger.error(" [FAIL] frontmatter %s: missing key '%s'", path, key)
errors += 1
if data.get("name") != path.parent.name:
logger.error(" [FAIL] frontmatter %s: name '%s' != directory '%s'",
path, data.get("name"), path.parent.name)
errors += 1
if not VERSION.match(str(data.get("version", ""))):
logger.error(" [FAIL] frontmatter %s: version '%s' is not x.y.z",
path, data.get("version"))
errors += 1
if not CATEGORY.match(str(data.get("category", ""))):
logger.error(" [FAIL] frontmatter %s: category '%s' is not kebab",
path, data.get("category"))
errors += 1
return errors
def check_skills_dir() -> int:
"""Every skill/<name>/ directory must contain a SKILL.md."""
errors = 0
for entry in sorted(SKILL_ROOT.iterdir()) if SKILL_ROOT.is_dir() else []:
if entry.is_dir() and not (entry / "SKILL.md").is_file():
logger.error(" [FAIL] skill '%s' has no SKILL.md", entry.name)
errors += 1
return errors
def check_markdown(paths: list[pathlib.Path]) -> int:
"""Markdownlint failure count, minus MARKDOWN_IGNORE; YAML front matter allowed."""
errors = 0
for path in paths:
api = PyMarkdownApi().enable_extension_by_identifier("front-matter")
# a 'title' key in front matter is not an implicit second h1 (SPEC item 9.6)
api.set_string_property("plugins.md025.front_matter_title", "^$")
for failure in api.scan_path(str(path)).scan_failures:
if failure.rule_id.lower() in MARKDOWN_IGNORE:
continue
errors += 1
logger.error(" [FAIL] markdown %s:%s [%s] %s", path, failure.line_number,
failure.rule_id, failure.rule_description)
return errors
def check_line_ending(path: pathlib.Path) -> int:
"""Text files must be LF, CRLF_SUFFIXES files CRLF; binary files are skipped."""
data = path.read_bytes()
if b"\0" in data:
return 0
if path.suffix.lower() in CRLF_SUFFIXES:
if b"\n" not in data.replace(b"\r\n", b""):
return 0
logger.error(" [FAIL] %s needs CRLF line endings (cmd.exe)", path)
return 1
if b"\r" not in data:
return 0
logger.error(" [FAIL] CRLF line endings: %s", path)
return 1
def check_private_key(path: pathlib.Path) -> int:
"""Return 1 when the file contains a PEM or OpenSSH private key."""
if PRIVATE_KEY.search(path.read_bytes()) is None:
return 0
logger.error(" [FAIL] private key must not be committed: %s", path)
return 1
def check_file_size(path: pathlib.Path) -> int:
"""Return 1 when the file is larger than MAX_FILE_SIZE bytes."""
size = path.stat().st_size
if size <= MAX_FILE_SIZE:
return 0
logger.error(" [FAIL] %s is %d bytes (limit %d)", path, size, MAX_FILE_SIZE)
return 1
def run_sanity() -> int:
"""Line endings, private keys, and file size on every repo file."""
checks = (check_line_ending, check_private_key, check_file_size)
all_files = iter_files(REPO_ROOT, ("",))
return sum(check(path) for path in all_files for check in checks)
def run_shellcheck() -> int:
"""shellcheck every shell script, including suffixless launchers."""
candidates = iter_files(REPO_ROOT, (*SHELL_SUFFIXES, ""))
return sum(check_shell(path) for path in candidates if is_shell_script(path))
def run_frontmatter() -> int:
"""Front matter and SKILL.md presence checks on every skill."""
if not SKILL_ROOT.is_dir():
return 0
skills = sorted(SKILL_ROOT.glob("*/SKILL.md"))
return check_skills_dir() + sum(check_frontmatter(path) for path in skills)
def run_each(check, suffixes: tuple[str, ...]) -> int:
"""Apply a per-file check to every repo file with a suffix."""
return sum(check(path) for path in iter_files(REPO_ROOT, suffixes))
def run_on_all(check, suffixes: tuple[str, ...]) -> int:
"""Apply a list check to every repo file with a suffix."""
return check(iter_files(REPO_ROOT, suffixes))
def run_pytest() -> int:
"""Run pytest in a subprocess so coverage is not skewed by imports here."""
cmd = [sys.executable, "-m", "pytest", "--cov-report=term-missing:skip-covered",
f"--cov-fail-under={MIN_COVERAGE_PERCENT}", "--cov=phronesis"]
return subprocess.run(cmd, cwd=REPO_ROOT, check=False).returncode
CHECKS = {
"sanity": run_sanity,
"python_syntax": functools.partial(run_each, check_python_syntax, (".py",)),
"pycodestyle": functools.partial(run_on_all, check_pycodestyle, (".py",)),
"shellcheck": run_shellcheck,
"frontmatter": run_frontmatter,
"markdown": functools.partial(run_on_all, check_markdown, (".md",)),
"pytest": run_pytest,
}
def timed(check) -> tuple[int, float]:
"""Run a check and return its error count and elapsed seconds."""
start = time.monotonic()
return check(), time.monotonic() - start
def main(names: list[str]) -> int:
"""Run the named checks in parallel, print a summary, return 1 on any failure."""
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {name: executor.submit(timed, CHECKS[name]) for name in names}
results = {name: future.result() for name, future in futures.items()}
print("\n########################################")
print("autotest results")
print("########################################")
for name, (errors, elapsed) in results.items():
print(f"[ {'PASS' if errors == 0 else 'FAIL'} ] {elapsed:6.2f}s {name}")
return int(any(errors != 0 for errors, _ in results.values()))
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("checks", nargs="*", metavar="CHECK",
help=f"checks to run (default: all): {', '.join(CHECKS)}")
selected = parser.parse_args().checks or list(CHECKS)
unknown = sorted(set(selected) - set(CHECKS))
if unknown:
parser.error(f"unknown check(s): {', '.join(unknown)}")
logging.basicConfig(level=logging.INFO, format="%(levelname)-5s %(message)s")
sys.exit(main(selected))