Files
fiona-assistant/src/fiona/run_tests.py
T
2026-09-29 16:01:16 +02:00

650 lines
25 KiB
Python

"""
Test Runner for Fiona tools.
This script provides an OOP interface for running tests, making it reusable
for both manual execution and CI/CD pipelines.
"""
from __future__ import annotations
import argparse
import logging
import os
import pathlib
import re
import subprocess
import sys
import tempfile
import xml.etree.ElementTree as ET
logger = logging.getLogger(__name__)
class TestRunner:
"""A test runner for Fiona tools that can be used both manually and in CI."""
__test__ = False
def __init__(self, base_dir: str | None = None):
"""
Initialize the test runner.
Args:
base_dir: Base directory for tests. Defaults to current directory.
"""
self.base_dir = base_dir or "."
self.test_results = {}
self.total_collected = 0
self.total_passed = 0
self.total_failed = 0
self.coverage_percent = None
def _sanitize_pytest_env(self) -> dict[str, str]:
"""Sanitize pytest-related environment variables to ensure proper output.
Removes options that suppress the short summary from PYTEST_ADDOPTS,
and unsets color-forcing variables that can interfere with output parsing.
Returns:
Dictionary of the original environment variable values that were modified.
"""
original_env: dict[str, str] = {}
# Save and sanitize PYTEST_ADDOPTS
pytest_addopts = os.environ.get("PYTEST_ADDOPTS", "")
if pytest_addopts:
# Remove --no-summary options
sanitized = re.sub(r"--no-summary\s*", "", pytest_addopts)
sanitized = re.sub(r"\s+--no-summary", "", sanitized)
sanitized = re.sub(r"\s+--no-summary\s+", " ", sanitized)
# Clean up extra spaces
sanitized = re.sub(r"\s+", " ", sanitized).strip()
if sanitized != pytest_addopts:
original_env["PYTEST_ADDOPTS"] = pytest_addopts
os.environ["PYTEST_ADDOPTS"] = sanitized
# Save and unset color-forcing variables
for var in ("FORCE_COLOR", "PY_COLORS"):
if var in os.environ:
original_env[var] = os.environ[var]
del os.environ[var]
return original_env
def _restore_pytest_env(self, original_env: dict[str, str]) -> None:
"""Restore the original pytest environment variables.
Args:
original_env: Dictionary of original environment variable values.
"""
for var, value in original_env.items():
if value:
os.environ[var] = value
else:
if var in os.environ:
del os.environ[var]
def _parse_pytest_output(self, output: str) -> dict:
"""Parse pytest output to extract test counts.
The counts come from the final summary line, so the last match
wins: assertion diffs printed by failing tests can contain
earlier `N passed` strings that must not be counted. Errored
tests (fixture or collection errors) count as failures so the
totals match the per-file listing.
"""
def last_count(pattern: str) -> int:
matches = re.findall(pattern, output)
return int(matches[-1]) if matches else 0
return {
"passed": last_count(r'(\d+) passed'),
"failed": last_count(r'(\d+) failed') + last_count(r'(\d+) errors?'),
"skipped": last_count(r'(\d+) skipped'),
}
def _parse_coverage_output(self, output: str) -> float:
"""Parse coverage output to extract percentage."""
match = re.search(r'TOTAL\s+\S+\s+\S+\s+(\d+)%', output)
if match:
return float(match.group(1))
match = re.search(r'Total coverage:\s+(\d+)%', output)
if match:
return float(match.group(1))
return 0.0
def _parse_coverage_files_output(self, output: str) -> dict[str, float]:
"""Parse the coverage report table into per-file percentages.
Matches the rows of `pytest --cov-report=term`
(e.g. `src/wiki/wiki.py 49 2 96%`); the `TOTAL`
row is excluded.
Args:
output: The pytest coverage output text.
Returns:
Dictionary mapping source file paths to their coverage
percentage.
"""
file_coverage: dict[str, float] = {}
pattern = re.compile(r"^(\S+\.py)\s+\d+\s+\d+\s+([\d.]+)%$", re.MULTILINE)
for match in pattern.finditer(output):
file_coverage[match.group(1)] = float(match.group(2))
return file_coverage
def _parse_failed_tests_output(self, output: str) -> list[dict]:
"""Parse the pytest output into failed and errored test details.
Combines the short summary (test id and exception line) with the
FAILURES traceback section (assert location and explanation) so
reports show what failed, where, and why. Errored tests (fixture
or collection errors, reported as `ERROR` lines) are included
with their id and message so they do not vanish from the report.
Failures are paired with their traceback blocks by order.
Args:
output: The pytest output text.
Returns:
List of dictionaries with the failed test `id`, the failure
`message` (exception line), the `location` (`file:line` of
the failing assert), and the `assert` explanation. When the
short summary is absent, failing tests are recovered from
the verbose progress lines so they still appear in reports.
"""
failures: list[dict] = []
errors: list[dict] = []
failed_pattern = re.compile(r"^FAILED (\S+) - (.+)$", re.MULTILINE)
error_pattern = re.compile(r"^ERROR (\S+) - (.+)$", re.MULTILINE)
for match in failed_pattern.finditer(output):
failures.append({
"id": match.group(1),
"message": match.group(2),
"location": None,
"assert": None,
})
for match in error_pattern.finditer(output):
errors.append({
"id": match.group(1),
"message": match.group(2),
"location": None,
"assert": None,
})
# traceback details only exist for FAILURES section entries, so
# they pair with the FAILED entries, never with the errored ones
details = self._parse_failures_section(output)
if len(details) == len(failures):
for failure, detail in zip(failures, details):
failure["location"] = detail["location"]
failure["assert"] = detail["assert"]
# fallback: when the short summary is missing (custom pytest
# addopts, plugin interference), the verbose progress lines still
# name every failing test; add any that were not already parsed
# so a failure can never silently vanish from the report
seen_ids = {entry["id"] for entry in failures + errors}
verbose_pattern = re.compile(
r"^(\S+\.py)::(\S+)\s+(FAILED|ERROR)\b", re.MULTILINE
)
for match in verbose_pattern.finditer(output):
entry_id = f"{match.group(1)}::{match.group(2)}"
if entry_id in seen_ids:
continue
outcome = match.group(3)
message = (
"test failed (no short summary in pytest output)"
if outcome == "FAILED"
else "test errored (no short summary in pytest output)"
)
(failures if outcome == "FAILED" else errors).append({
"id": entry_id,
"message": message,
"location": None,
"assert": None,
})
return failures + errors
def _parse_failures_section(self, output: str) -> list[dict]:
"""Parse the FAILURES traceback section into per-test details.
Args:
output: The pytest output text.
Returns:
List of dictionaries (in test order) with the `location`
(`file:line` of the failing statement) and the `assert`
explanation: the first non-blank pytest E-line with any
leading `AssertionError:` prefix removed (e.g.
`assert 1 == 2`), or the failing statement when the
traceback has no E-lines.
"""
section_match = re.search(
r"^=+ FAILURES =+\n(.*?)(?=^=+ |\Z)", output, re.MULTILINE | re.DOTALL
)
if not section_match:
return []
section = section_match.group(1)
parts = re.split(
r"^_{5,} (.+?) _{5,}$", section, flags=re.MULTILINE
)
details = []
for i in range(1, len(parts) - 1, 2):
body = parts[i + 1]
location = None
statement = None
explanation = None
for line in body.splitlines():
loc = re.match(r"^(\S+\.py):(\d+): ", line)
if loc:
location = f"{loc.group(1)}:{loc.group(2)}"
if line.startswith(">"):
statement = line[1:].strip()
if explanation is None:
e_match = re.match(r"^E\s+(.*)$", line)
if e_match and e_match.group(1).strip():
explanation = re.sub(
r"^AssertionError:\s*", "", e_match.group(1).strip()
)
details.append({
"location": location,
"assert": explanation or statement,
})
return details
def _parse_test_files_output(self, output: str) -> dict[str, dict]:
"""Parse pytest verbose output into per-file test counts.
Matches the progress lines of `pytest -v` (e.g.
`src/tests/test_a.py::TestX::test_one PASSED [ 10%]`); the
trailing `FAILED <path>` summary lines are ignored so failures
are not counted twice.
Args:
output: The pytest verbose output text.
Returns:
Dictionary mapping test file names to their
passed/failed/skipped/total counts.
"""
file_results: dict[str, dict] = {}
pattern = re.compile(
r"^(\S+\.py)::\S+\s+(PASSED|FAILED|ERROR|SKIPPED)\b", re.MULTILINE
)
for match in pattern.finditer(output):
file_name = pathlib.Path(match.group(1)).name
outcome = match.group(2)
counts = file_results.setdefault(
file_name, {"passed": 0, "failed": 0, "skipped": 0, "total": 0}
)
if outcome == "PASSED":
counts["passed"] += 1
elif outcome == "SKIPPED":
counts["skipped"] += 1
else:
counts["failed"] += 1
counts["total"] += 1
return file_results
def _parse_junit_failures(self, junit_path: str | pathlib.Path) -> list[dict]:
"""Parse the pytest JUnit XML report into failed/errored details.
The JUnit report is a machine-readable source for failure
details that does not depend on pytest's terminal formatting,
so the report stays correct even when the captured output lacks
the short summary or tracebacks (custom addopts, plugins,
colors).
Args:
junit_path: Path to the JUnit XML report (xunit1 family,
so testcases carry `file` and `line` attributes).
Returns:
List of dictionaries with the failed test `id`, `message`
(first line of the reported exception, with pytest's
`failed on setup/teardown with "..."` wrapper removed),
`location` (`file:line`), and the `assert` explanation
(message with any `AssertionError:` prefix removed, None for
other errors). Empty when the report is missing or
unreadable.
"""
try:
tree = ET.parse(junit_path)
except (OSError, ET.ParseError):
logger.debug("no readable junit xml report at %s", junit_path)
return []
entries: list[dict] = []
for case in tree.iter("testcase"):
outcome = next(
(child for child in case if child.tag in ("failure", "error")),
None,
)
if outcome is None:
continue
file_name = case.get("file")
name = case.get("name")
if not file_name or not name:
continue
class_parts = self._class_parts_from_junit(
case.get("classname", ""), file_name
)
message = (outcome.get("message") or "").split("\n")[0] or "test failed"
setup_match = re.fullmatch(
r'failed on (?:setup|teardown) with "(.*)"', message
)
if setup_match and setup_match.group(1):
message = setup_match.group(1)
assertion = None
if message.startswith("AssertionError:"):
assertion = message[len("AssertionError:"):].strip() or None
line_number = case.get("line")
entries.append({
"id": "::".join([file_name, *class_parts, name]),
"message": message,
"location": f"{file_name}:{line_number}" if line_number else None,
"assert": assertion,
})
return entries
@staticmethod
def _class_parts_from_junit(classname: str, file_name: str) -> list[str]:
"""Extract the test class path from a junit testcase classname.
Args:
classname: The junit `classname` attribute, e.g.
`src.tests.test_a.TestX`.
file_name: The junit `file` attribute, e.g.
`src/tests/test_a.py`.
Returns:
The class path between the module and the test name (e.g.
`["TestX"]`), or an empty list for module-level tests.
"""
module = ".".join(pathlib.Path(file_name).with_suffix("").parts)
if classname == module:
return []
if classname.startswith(module + "."):
return classname[len(module) + 1:].split(".")
return []
@staticmethod
def _failure_key(entry_id: str) -> tuple[str, ...]:
"""Build a path-independent identity key for a failure entry id.
Args:
entry_id: The failure id, e.g. `path/to/test_a.py::TestX::t`.
Returns:
Tuple of the file basename and the `::`-separated id parts,
so the same test matches across path forms (pytest records
junit `file` paths relative to its rootdir).
"""
parts = entry_id.split("::")
return (pathlib.Path(parts[0]).name, *parts[1:])
@staticmethod
def _is_generic_fallback(entry: dict) -> bool:
"""Report whether a failure entry carries no real message.
Args:
entry: A failure detail dictionary.
Returns:
True when the entry came from the verbose fallback (its
message only notes the missing short summary), False when
it carries a real exception message.
"""
return entry.get("message", "").endswith("(no short summary in pytest output)")
def _merge_failure_details(self, regex_entries: list[dict],
junit_entries: list[dict]) -> list[dict]:
"""Merge junit failure details into the regex-parsed entries.
Regex-parsed entries match pytest's own display, so they win
when both sources describe the same test; missing fields
(location, assert) are filled from the junit entry. Junit
entries replace generic verbose-fallback entries (which carry
no real message) and add any test the regex parsing missed.
Entries are matched by file basename and test name, so recorded
path differences do not create duplicates.
Args:
regex_entries: Failure entries parsed from pytest output.
junit_entries: Failure entries parsed from the junit report.
Returns:
The merged failure entry list.
"""
merged = list(regex_entries)
for junit_entry in junit_entries:
key = self._failure_key(junit_entry["id"])
match_index = next(
(
i for i, entry in enumerate(merged)
if self._failure_key(entry["id"]) == key
),
None,
)
if match_index is None:
merged.append(junit_entry)
elif self._is_generic_fallback(merged[match_index]):
merged[match_index] = junit_entry
else:
for field in ("location", "assert", "message"):
if not merged[match_index].get(field) and junit_entry.get(field):
merged[match_index][field] = junit_entry[field]
return merged
def run_pytest(self, test_path: str, verbose: bool = False,
capture_output: bool = False, with_coverage: bool = True,
ignore: str | None = None) -> dict:
"""
Run pytest on the specified test path in a subprocess.
Running pytest in a fresh interpreter keeps results and coverage
independent of the modules the calling process has already imported.
Args:
test_path: Path to the test file or directory
verbose: Whether to run in verbose mode
capture_output: Whether to suppress the pytest output (for CI)
with_coverage: Whether to run with coverage (default: True)
ignore: Pattern to ignore (e.g., "test_sanity.py")
Returns:
Dictionary containing test results and metadata
"""
# Sanitize pytest environment to ensure proper output parsing
original_env = self._sanitize_pytest_env()
cmd = [sys.executable, "-m", "pytest", test_path]
if verbose:
cmd.append("-v")
if with_coverage:
cmd.extend(["--cov=src/fiona", "--cov-report=term"])
if ignore:
cmd.extend(["--ignore-glob", f"*{ignore}*"])
# machine-readable failure details that survive terminal
# formatting quirks (missing summary, colors, plugin output)
junit_fd, junit_path = tempfile.mkstemp(suffix=".xml", prefix="fiona-junit-")
os.close(junit_fd)
cmd.extend(["--junitxml", junit_path, "-o", "junit_family=xunit1"])
# pin the output format the parsers expect: color-forcing
# environment variables (PY_COLORS, FORCE_COLOR) would break them
cmd.append("--color=no")
# Set PYTHONPATH to include src so the fiona package resolves
env = os.environ.copy()
src_dir = str(pathlib.Path(__file__).resolve().parent.parent)
if os.path.exists(src_dir):
current_pythonpath = env.get("PYTHONPATH", "")
if current_pythonpath:
env["PYTHONPATH"] = f"{src_dir}:{current_pythonpath}"
else:
env["PYTHONPATH"] = src_dir
try:
completed = subprocess.run(cmd, capture_output=True, text=True, check=False, env=env)
if not capture_output:
print(completed.stdout)
logger.debug("pytest command: %s", " ".join(str(part) for part in cmd))
logger.debug("pytest return code: %s", completed.returncode)
counts = self._parse_pytest_output(completed.stdout)
coverage_percent = self._parse_coverage_output(completed.stdout) if with_coverage else None
self.coverage_percent = coverage_percent
failed_tests = self._parse_failed_tests_output(completed.stdout)
junit_failures = self._parse_junit_failures(junit_path)
if junit_failures:
if not any(not self._is_generic_fallback(entry)
for entry in failed_tests):
logger.warning(
"pytest short summary missing from output; failure "
"details come from the junit report (check "
"PYTEST_ADDOPTS, --no-summary, or plugin output)"
)
failed_tests = self._merge_failure_details(failed_tests, junit_failures)
test_results = {
"return_code": completed.returncode,
"passed_count": counts["passed"],
"failed_count": counts["failed"],
"skipped_count": counts["skipped"],
"total_count": counts["passed"] + counts["failed"] + counts["skipped"],
"coverage_percent": coverage_percent,
"file_results": self._parse_test_files_output(completed.stdout),
"coverage_files": self._parse_coverage_files_output(completed.stdout),
"failed_tests": failed_tests,
"passed": completed.returncode == 0,
}
self.test_results[test_path] = test_results
if test_results["failed_count"] and not test_results["failed_tests"]:
logger.warning(
"summary reports %d failed tests but no failure details "
"were parsed; rerun with LOG_LEVEL=DEBUG for the raw "
"pytest output",
test_results["failed_count"],
)
logger.debug("raw pytest output:\n%s", completed.stdout)
self.total_collected += test_results["passed_count"] + test_results["failed_count"]
self.total_passed += test_results["passed_count"]
self.total_failed += test_results["failed_count"]
return self.test_results[test_path]
except (OSError, subprocess.SubprocessError) as e:
self.test_results[test_path] = {
"return_code": -1,
"passed_count": None,
"failed_count": None,
"skipped_count": None,
"total_count": None,
"coverage_percent": None,
"passed": False,
"exc": e
}
return self.test_results[test_path]
finally:
pathlib.Path(junit_path).unlink(missing_ok=True)
# Restore the original pytest environment
self._restore_pytest_env(original_env)
def run_all_tests(self, verbose: bool = False, capture_output: bool = False,
with_coverage: bool = True) -> bool:
"""
Run all tests for Fiona tools.
Args:
verbose: Whether to run in verbose mode
capture_output: Whether to capture output (for CI)
with_coverage: Whether to run with coverage (default: True)
Returns:
True if all tests passed, False otherwise
"""
test_paths = [
"tests"
]
all_passed = True
for test_path in test_paths:
result = self.run_pytest(test_path, verbose, capture_output, with_coverage)
if not result["passed"]:
all_passed = False
return all_passed
def get_summary(self) -> dict:
"""
Get a summary of all test results.
Returns:
Dictionary containing summary of test results
"""
return {
"total_tests": self.total_collected,
"passed_tests": self.total_passed,
"failed_tests": self.total_failed,
"coverage_percent": self.coverage_percent,
"test_details": self.test_results
}
def main():
"""Main function for command-line execution."""
parser = argparse.ArgumentParser(description="Run Fiona tools tests.")
parser.add_argument("--verbose", action="store_true", help="Run tests in verbose mode")
parser.add_argument("--ci", action="store_true", help="Run in CI mode (capture output)")
parser.add_argument("--test", help="Run specific test file or directory")
parser.add_argument("--skip-cov", action="store_true", help="Skip coverage")
args = parser.parse_args()
# Initialize test runner
runner = TestRunner()
# Determine if we should run with coverage
with_coverage = not args.skip_cov
if args.test:
# Run specific test
result = runner.run_pytest(args.test, args.verbose, args.ci, with_coverage)
if not result["passed"]:
print(f"Test failed: {args.test}")
sys.exit(1)
else:
# Run all tests
all_passed = runner.run_all_tests(args.verbose, args.ci, with_coverage)
if not all_passed:
print("Some tests failed") #TODO: this is not very informative, need improving
sys.exit(1)
# Print summary
summary = runner.get_summary()
print("\nTest Summary:")
print(f"Passed: {summary['passed_tests']}/{summary['total_tests']}")
if summary['coverage_percent'] is not None:
print(f"Coverage: {summary['coverage_percent']}%")
if summary['failed_tests'] > 0:
print(f"Failed: {summary['failed_tests']}")
sys.exit(1)
print("All tests passed!")
if __name__ == "__main__":
main()