tests/validate_native_reports.py

   1 #!/usr/bin/env python3
   2 """Validate FriiDump native report fixtures against the v1 contract."""
   3
   4 from __future__ import annotations
   5
   6 import argparse
   7 import json
   8 import subprocess
   9 import sys
  10 import uuid
  11 from pathlib import Path
  12 from typing import Any
  13
  14
  15 def load_json(path: Path) -> Any:
  16     try:
  17         return json.loads(path.read_text(encoding="utf-8"))
  18     except (OSError, UnicodeError, json.JSONDecodeError) as exc:
  19         raise RuntimeError(f"Could not decode {path}: {exc}") from exc
  20
  21
  22 def require(condition: bool, message: str) -> None:
  23     if not condition:
  24         raise RuntimeError(message)
  25
  26
  27 def semantic_fixture_checks(path: Path, report: dict[str, Any]) -> None:
  28     run = report["run"]
  29     dump = report["dump"]
  30     notes = report["notes"]
  31
  32     parsed_uuid = uuid.UUID(run["run_id"])
  33     require(parsed_uuid.int != 0, f"{path.name}: nil run UUID")
  34
  35     scope_notes = [
  36         note for note in notes
  37         if isinstance(note, str) and note.lower().startswith("measurement scope:")
  38     ]
  39
  40     if dump["attempted"]:
  41         require(len(scope_notes) == 1,
  42                 f"{path.name}: attempted dump must have exactly one measurement scope")
  43
  44     artifacts = report["artifacts"]
  45     dump_artifacts = [
  46         artifact for artifact in artifacts
  47         if isinstance(artifact, dict) and artifact.get("type") == "dump_output"
  48     ]
  49
  50     if dump["attempted"] and dump.get("output_path"):
  51         require(len(dump_artifacts) == 1,
  52                 f"{path.name}: attempted dump with output must have one dump_output artifact")
  53         artifact = dump_artifacts[0]
  54         require(artifact["path"] == dump["output_path"],
  55                 f"{path.name}: dump_output path mismatch")
  56         if dump.get("byte_count") is not None:
  57             require(artifact["bytes"] == dump["byte_count"],
  58                     f"{path.name}: dump_output byte count mismatch")
  59         if report["hashes"].get("sha256") is not None:
  60             require(artifact["sha256"] == report["hashes"]["sha256"],
  61                     f"{path.name}: dump_output SHA-256 mismatch")
  62
  63     if path.name == "success-gamecube.friidump.json":
  64         require(run == {**run, "test_type": "full_dump", "result": "pass"},
  65                 f"{path.name}: success outcome mismatch")
  66         require("Measurement scope: full_optical_payload." in notes,
  67                 f"{path.name}: optical scope missing")
  68
  69     if path.name == "success-xbox-assembled.friidump.json":
  70         require("Measurement scope: assembled_output." in notes,
  71                 f"{path.name}: assembled-output scope missing")
  72         require(report["drive"]["firmware_modified"] is True,
  73                 f"{path.name}: modified-firmware identity missing")
  74         require(report["media"]["region"] == "North America",
  75                 f"{path.name}: Xbox XBE region mismatch")
  76
  77     if path.name == "modified-firmware.friidump.json":
  78         require("UTF-8 preservation fixture: Pokémon." in notes,
  79                 f"{path.name}: UTF-8 text was not preserved")
  80
  81     if path.name == "partial-cancelled.friidump.json":
  82         require(run["test_type"] == "partial_dump" and run["result"] == "partial",
  83                 f"{path.name}: cancellation outcome mismatch")
  84         require(dump["result"] == "partial" and
  85                 dump["failure_stage"] == "user_cancelled",
  86                 f"{path.name}: cancellation dump state mismatch")
  87         require("Measurement scope: partial_progress." in notes,
  88                 f"{path.name}: cancellation scope missing")
  89
  90     if path.name == "seed-failure.friidump.json":
  91         require(run["test_type"] == "seed_only" and run["result"] == "fail",
  92                 f"{path.name}: seed-failure outcome mismatch")
  93         require(dump["result"] == "not_attempted",
  94                 f"{path.name}: seed failure must not claim a dump")
  95
  96     if path.name == "no-media.friidump.json":
  97         require(run["test_type"] == "diagnostic_no_media" and
  98                 run["result"] == "not_applicable",
  99                 f"{path.name}: no-media outcome mismatch")
 100
 101
 102 def run_php_validator(php_validator: Path, report_path: Path) -> tuple[int, int]:
 103     result = subprocess.run(
 104         ["php", str(php_validator), str(report_path)],
 105         check=False,
 106         capture_output=True,
 107         text=True,
 108     )
 109     if result.returncode not in (0, 1):
 110         raise RuntimeError(
 111             f"PHP validator failed for {report_path}:\n{result.stdout}\n{result.stderr}"
 112         )
 113     try:
 114         decoded = json.loads(result.stdout)
 115     except json.JSONDecodeError as exc:
 116         raise RuntimeError(
 117             f"PHP validator returned invalid JSON for {report_path}: {exc}"
 118         ) from exc
 119     if not decoded.get("valid"):
 120         raise RuntimeError(
 121             f"PHP validator rejected {report_path}: {decoded.get('errors')}"
 122         )
 123     return len(decoded.get("errors", [])), len(decoded.get("warnings", []))
 124
 125
 126 def main() -> int:
 127     parser = argparse.ArgumentParser()
 128     parser.add_argument("--fixtures", type=Path, required=True)
 129     parser.add_argument(
 130         "--schema",
 131         type=Path,
 132         default=Path(__file__).parent / "contracts" /
 133                 "friidump-test-result.v1.schema.json",
 134     )
 135     parser.add_argument("--php-validator", type=Path)
 136     parser.add_argument(
 137         "--allow-arbitrary",
 138         action="store_true",
 139         help="Validate any report set without requiring the built-in fixture filenames",
 140     )
 141     args = parser.parse_args()
 142
 143     try:
 144         from jsonschema import Draft202012Validator
 145     except ImportError as exc:
 146         raise RuntimeError(
 147             "The Python 'jsonschema' package is required for strict contract validation"
 148         ) from exc
 149
 150     schema = load_json(args.schema)
 151     validator = Draft202012Validator(schema)
 152     reports = sorted(args.fixtures.rglob("*.friidump.json"))
 153     require(bool(reports), f"No .friidump.json fixtures found under {args.fixtures}")
 154
 155     seen_run_ids: set[str] = set()
 156     php_warning_total = 0
 157
 158     for path in reports:
 159         report = load_json(path)
 160         errors = sorted(validator.iter_errors(report), key=lambda item: list(item.path))
 161         if errors:
 162             rendered = "\n".join(
 163                 f"  {list(error.path)}: {error.message}" for error in errors
 164             )
 165             raise RuntimeError(f"JSON Schema rejected {path}:\n{rendered}")
 166
 167         run_id = report["run"]["run_id"]
 168         require(run_id not in seen_run_ids, f"Duplicate fixture run UUID: {run_id}")
 169         seen_run_ids.add(run_id)
 170         semantic_fixture_checks(path, report)
 171
 172         php_summary = ""
 173         if args.php_validator:
 174             errors_count, warnings_count = run_php_validator(args.php_validator, path)
 175             php_warning_total += warnings_count
 176             php_summary = f", php errors={errors_count}, warnings={warnings_count}"
 177
 178         print(f"PASS {path.relative_to(args.fixtures)}: schema + fixture semantics{php_summary}")
 179
 180     if not args.allow_arbitrary:
 181         expected_names = {
 182             "success-gamecube.friidump.json",
 183             "success-xbox-assembled.friidump.json",
 184             "partial-gamecube.friidump.json",
 185             "partial-cancelled.friidump.json",
 186             "seed-failure.friidump.json",
 187             "no-media.friidump.json",
 188             "modified-firmware.friidump.json",
 189             "identity-no-overwrite.friidump.json",
 190         }
 191         observed_names = {path.name for path in reports}
 192         missing = expected_names - observed_names
 193         require(not missing, f"Required fixture reports are missing: {sorted(missing)}")
 194
 195     print()
 196     print(f"Native report validation: PASS ({len(reports)} reports)")
 197     if args.php_validator:
 198         print(f"Database semantic validator warnings: {php_warning_total}")
 199         print("The single expected warning is the honest seed-failure fixture without a disc title.")
 200     return 0
 201
 202
 203 if __name__ == "__main__":
 204     try:
 205         raise SystemExit(main())
 206     except RuntimeError as exc:
 207         print(f"FAIL: {exc}", file=sys.stderr)
 208         raise SystemExit(1)