"""Visual testing utilities — pixel assertions for headless Vulkan rendering.
All functions operate on numpy arrays (H, W, 4) uint8 in RGBA order,
as returned by Renderer.capture_frame(). No external dependencies beyond numpy.
Usage::
from simvx.graphics.testing import assert_pixel, assert_not_blank
pixels = app.run_headless(scene, frames=5, capture_frames=[4])[0]
assert_not_blank(pixels)
assert_pixel(pixels, 160, 120, (255, 0, 0, 255), tolerance=10)
"""
from __future__ import annotations
import base64
import json
import logging
from datetime import UTC, datetime
from pathlib import Path
import numpy as np
log = logging.getLogger(__name__)
__all__ = [
"assert_pixel",
"assert_colour_ratio",
"assert_region_colour",
"assert_no_colour",
"assert_not_blank",
"colour_ratio",
"save_png",
"save_image",
"save_diff_image",
"ScreenshotReport",
"VisualRegression",
]
# ---------------------------------------------------------------------------
# Pixel distance helpers
# ---------------------------------------------------------------------------
def _colour_dist(pixels: np.ndarray, colour: tuple[int, ...]) -> np.ndarray:
"""Per-pixel max-channel distance to *colour* (H, W) uint8."""
ref = np.array(colour[:3], dtype=np.int16)
return np.abs(pixels[:, :, :3].astype(np.int16) - ref).max(axis=2)
# ---------------------------------------------------------------------------
# Pixel assertions
# ---------------------------------------------------------------------------
[docs]
def assert_pixel(
pixels: np.ndarray,
x: int,
y: int,
expected_rgba: tuple[int, ...],
tolerance: int = 2,
) -> None:
"""Assert the pixel at (x, y) matches *expected_rgba* within *tolerance* per channel."""
actual = pixels[y, x]
diff = np.abs(actual.astype(int) - np.array(expected_rgba, dtype=int))
if diff.max() > tolerance:
raise AssertionError(
f"Pixel ({x},{y}): expected {expected_rgba}, got {tuple(actual)}, "
f"max channel diff {diff.max()} > tolerance {tolerance}"
)
[docs]
def colour_ratio(
pixels: np.ndarray,
colour: tuple[int, ...],
tolerance: int = 10,
) -> float:
"""Return the fraction of pixels matching *colour* within *tolerance*."""
dist = _colour_dist(pixels, colour)
return float((dist <= tolerance).sum()) / (pixels.shape[0] * pixels.shape[1])
[docs]
def assert_colour_ratio(
pixels: np.ndarray,
colour: tuple[int, ...],
expected_ratio: float,
tolerance: float = 0.02,
colour_tolerance: int = 10,
) -> None:
"""Assert approximately *expected_ratio* of pixels match *colour*."""
actual = colour_ratio(pixels, colour, colour_tolerance)
if abs(actual - expected_ratio) > tolerance:
raise AssertionError(
f"Colour {colour}: expected ratio {expected_ratio:.3f} ± {tolerance:.3f}, " f"got {actual:.3f}"
)
[docs]
def assert_region_colour(
pixels: np.ndarray,
rect: tuple[int, int, int, int],
expected_colour: tuple[int, ...],
tolerance: int = 5,
) -> None:
"""Assert all pixels in rect (x, y, w, h) match *expected_colour*."""
x, y, w, h = rect
region = pixels[y : y + h, x : x + w]
dist = _colour_dist(region, expected_colour)
bad = (dist > tolerance).sum()
if bad > 0:
worst = int(dist.max())
raise AssertionError(
f"Region ({x},{y},{w},{h}): {bad} pixels differ from {expected_colour}, "
f"worst channel diff {worst} > tolerance {tolerance}"
)
[docs]
def assert_no_colour(
pixels: np.ndarray,
colour: tuple[int, ...],
tolerance: int = 5,
) -> None:
"""Assert no pixel matches *colour*."""
dist = _colour_dist(pixels, colour)
matches = (dist <= tolerance).sum()
if matches > 0:
raise AssertionError(f"Found {matches} pixels matching {colour} (tolerance {tolerance})")
[docs]
def assert_not_blank(pixels: np.ndarray) -> None:
"""Assert the image is not a single solid colour."""
flat = pixels.reshape(-1, 4)
if np.all(flat == flat[0]):
raise AssertionError(f"Image is blank — every pixel is {tuple(flat[0])}")
# ---------------------------------------------------------------------------
# PNG I/O — re-exported from assets.image_loader
# ---------------------------------------------------------------------------
from simvx.graphics.assets.image_loader import _load_png, _png_chunk, save_png # noqa: F401
# ---------------------------------------------------------------------------
# Image save/diff helpers
# ---------------------------------------------------------------------------
[docs]
def save_image(path: str | Path, pixels: np.ndarray) -> None:
"""Save RGBA pixels to file. Auto-detects format from extension (.png or .ppm)."""
p = Path(path)
if p.suffix.lower() == ".ppm":
h, w = pixels.shape[:2]
with open(p, "wb") as f:
f.write(f"P6\n{w} {h}\n255\n".encode())
f.write(pixels[:, :, :3].tobytes())
else:
save_png(p.with_suffix(".png") if not p.suffix else p, pixels)
[docs]
def save_diff_image(path: str | Path, actual: np.ndarray, expected: np.ndarray) -> None:
"""Save a visual diff image highlighting differences (5x amplified)."""
diff = np.abs(actual[:, :, :3].astype(np.int16) - expected[:, :, :3].astype(np.int16))
diff = np.clip(diff * 5, 0, 255).astype(np.uint8)
rgba = np.concatenate([diff, np.full((*diff.shape[:2], 1), 255, dtype=np.uint8)], axis=2)
save_image(path, rgba)
# ---------------------------------------------------------------------------
# Screenshot report for CI/CD
# ---------------------------------------------------------------------------
[docs]
class ScreenshotReport:
"""Collects screenshots during a test run and produces CI-friendly output."""
def __init__(self, output_dir: Path):
self.output_dir = Path(output_dir)
self._screenshots: list[dict] = []
self._description: str = ""
[docs]
def capture(self, pixels: np.ndarray, label: str) -> Path:
"""Save frame as PNG, return path."""
self.output_dir.mkdir(parents=True, exist_ok=True)
idx = len(self._screenshots)
path = self.output_dir / f"{idx:03d}_{label}.png"
save_png(path, pixels)
h, w = pixels.shape[:2]
self._screenshots.append({"label": label, "path": str(path), "width": w, "height": h})
return path
[docs]
def capture_with_diff(self, actual: np.ndarray, expected: np.ndarray, label: str) -> Path:
"""Save actual + expected + diff as side-by-side composite PNG."""
self.output_dir.mkdir(parents=True, exist_ok=True)
diff = np.abs(actual[:, :, :3].astype(np.int16) - expected[:, :, :3].astype(np.int16))
diff = np.clip(diff * 5, 0, 255).astype(np.uint8)
diff_rgba = np.concatenate([diff, np.full((*diff.shape[:2], 1), 255, dtype=np.uint8)], axis=2)
composite = np.concatenate([actual, expected, diff_rgba], axis=1)
idx = len(self._screenshots)
path = self.output_dir / f"{idx:03d}_{label}_diff.png"
save_png(path, composite)
h, w = composite.shape[:2]
self._screenshots.append({"label": f"{label} (diff)", "path": str(path), "width": w, "height": h})
return path
[docs]
def add_description(self, text: str) -> None:
"""Attach a text description (e.g., from scene_describe())."""
self._description = text
[docs]
def finalize(self, test_name: str = "", status: str = "passed") -> Path:
"""Write report.json and report.html to output_dir. Return path to JSON."""
self.output_dir.mkdir(parents=True, exist_ok=True)
report = {
"test_name": test_name,
"status": status,
"screenshots": self._screenshots,
"description": self._description,
"timestamp": datetime.now(UTC).isoformat(),
}
json_path = self.output_dir / "report.json"
json_path.write_text(json.dumps(report, indent=2))
# Build self-contained HTML
html_parts = [
"<!DOCTYPE html><html><head><meta charset='utf-8'>",
"<title>Screenshot Report</title>",
"<style>body{font-family:monospace;margin:20px;background:#1a1a2e;colour:#e0e0e0}",
"img{max-width:100%;border:1px solid #444;margin:8px 0}",
".status-passed{colour:#4caf50}.status-failed{colour:#f44336}",
"pre{background:#0d0d1a;padding:12px;overflow-x:auto;border-radius:4px}</style></head><body>",
f"<h1>{test_name or 'Test Report'}</h1>",
f"<p class='status-{status}'>Status: {status}</p>",
]
if self._description:
html_parts.append(f"<h2>Description</h2><pre>{self._description}</pre>")
for ss in self._screenshots:
png_path = Path(ss["path"])
if png_path.exists():
b64 = base64.b64encode(png_path.read_bytes()).decode()
html_parts.append(f"<h3>{ss['label']}</h3>")
html_parts.append(f"<p>{ss['width']}x{ss['height']}</p>")
html_parts.append(f"<img src='data:image/png;base64,{b64}'>")
html_parts.append("</body></html>")
html_path = self.output_dir / "report.html"
html_path.write_text("".join(html_parts))
return json_path
# ---------------------------------------------------------------------------
# Visual regression testing
# ---------------------------------------------------------------------------
[docs]
class VisualRegression:
"""Manages baseline screenshots and compares rendered frames against them.
On first run (no baseline exists), saves the current frame as the baseline and passes.
On subsequent runs, compares against baseline and fails if pixels differ beyond threshold.
On failure, saves actual frame and diff image alongside baseline for inspection.
Usage::
def test_red_cube(capture, regression):
frames = capture(RedCubeScene(), frames=3, capture_frames=[2])
regression.assert_matches(frames[0], "red_cube")
"""
def __init__(self, baseline_dir: Path):
self.baseline_dir = baseline_dir
self._update_mode = False
def _baseline_path(self, name: str) -> Path:
return self.baseline_dir / f"{name}.png"
def _actual_path(self, name: str) -> Path:
return self.baseline_dir / f"{name}_actual.png"
def _diff_path(self, name: str) -> Path:
return self.baseline_dir / f"{name}_diff.png"
[docs]
def assert_matches(self, pixels: np.ndarray, name: str, threshold: float = 0.001) -> None:
"""Compare pixels against stored baseline."""
self.baseline_dir.mkdir(parents=True, exist_ok=True)
baseline = self._baseline_path(name)
if self._update_mode:
self.update_baseline(pixels, name)
return
if not baseline.exists():
save_png(baseline, pixels)
return
expected = _load_png(baseline)
if pixels.shape != expected.shape:
save_png(self._actual_path(name), pixels)
raise AssertionError(
f"Visual regression '{name}': shape mismatch — "
f"baseline {expected.shape} vs actual {pixels.shape}. "
f"Actual saved to {self._actual_path(name)}"
)
ratio = self.pixel_diff_ratio(pixels, expected)
if ratio > threshold:
save_png(self._actual_path(name), pixels)
save_diff_image(self._diff_path(name), pixels, expected)
raise AssertionError(
f"Visual regression '{name}': {ratio:.4%} pixels differ (threshold {threshold:.4%}). "
f"Baseline: {baseline}, Actual: {self._actual_path(name)}, Diff: {self._diff_path(name)}"
)
for p in (self._actual_path(name), self._diff_path(name)):
p.unlink(missing_ok=True)
[docs]
def update_baseline(self, pixels: np.ndarray, name: str) -> None:
"""Force-overwrite baseline (for intentional visual changes)."""
self.baseline_dir.mkdir(parents=True, exist_ok=True)
save_png(self._baseline_path(name), pixels)
for p in (self._actual_path(name), self._diff_path(name)):
p.unlink(missing_ok=True)
[docs]
@staticmethod
def pixel_diff_ratio(a: np.ndarray, b: np.ndarray, tolerance: int = 2) -> float:
"""Fraction of pixels that differ beyond tolerance per channel."""
diff = np.abs(a.astype(np.int16) - b.astype(np.int16))
exceeds = diff.max(axis=2) > tolerance
return float(exceeds.sum()) / (a.shape[0] * a.shape[1])