tmk241/mim
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

Add terminal E2E matrix runner

Browse Source
This commit is contained in:
slhx agent
2026-06-21 17:36:24 +02:00
parent 86af6d774e
commit 5d7f239d34
3 changed files with 216 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/terminal_e2e.py
+163 -64
View File
@@ -1,12 +1,13 @@
#!/usr/bin/env python3
"""PTY-backed terminal E2E harness for mim.
"""PTY-backed terminal E2E matrix for mim.
Runs a tiny high-signal scenario through a real pseudo-terminal and writes
Runs high-signal scenarios through a real pseudo-terminal and writes
browser-viewable terminal artifacts. No third-party dependencies.
"""
from __future__ import annotations
import argparse
import dataclasses
import hashlib
import html
import os
@@ -19,10 +20,28 @@ import sys
import tempfile
import time
from pathlib import Path
from typing import Callable
ESC = 0x1B
SCENARIO_ID = "raw-key-chrome-crlf"
TMP_RE = re.compile(r"/tmp/mim-terminal-e2e-[^/]+")
CSI_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
@dataclasses.dataclass(frozen=True)
class Scenario:
id: str
width: int
height: int
file_state: str
input_profile: str
mode_surface: str
color_mode: str
target_kind: str
setup: Callable[[Path], tuple[Path, str | None]]
payloads: tuple[bytes, ...]
expect_saved: str | None
required_raw: tuple[str, ...]
forbidden_raw: tuple[str, ...] = ("^[",)
def visible_controls(data: bytes) -> str:
@@ -55,6 +74,10 @@ def has_bare_lf(data: bytes) -> bool:
return any(byte == 0x0A and (i == 0 or data[i - 1] != 0x0D) for i, byte in enumerate(data))
def strip_csi(text: str) -> str:
return CSI_RE.sub("", text)
def self_test() -> None:
sample = b"a\nb\r\n\x1b[31m\t\x01"
visible = visible_controls(sample)
@@ -119,7 +142,6 @@ class TerminalGrid:
self.col = 0
elif command == "J" and params.endswith("2"):
self.clear()
# SGR and unknown CSI are intentionally ignored for semantic grid.
i = j + 1
continue
self.put_char(ch)
@@ -148,10 +170,12 @@ def read_available(fd: int, timeout: float = 0.12) -> bytes:
return b"".join(chunks)
def spawn_under_pty(argv: list[str], width: int, height: int) -> tuple[int, int]:
def spawn_under_pty(argv: list[str], width: int, height: int, color_mode: str, cwd: Path) -> tuple[int, int]:
pid, fd = pty.fork()
if pid == 0:
os.environ["TERM"] = "xterm-256color"
os.chdir(cwd)
os.environ["TERM"] = {"truecolor": "xterm-256color", "mono": "vt100"}.get(color_mode, "xterm-256color")
os.environ["COLORTERM"] = "truecolor" if color_mode == "truecolor" else ""
try:
import fcntl
import struct
@@ -173,36 +197,35 @@ def svg_for_lines(lines: list[str], width: int, height: int) -> str:
svg_h = pad * 2 + height * cell_h
body = []
for idx, line in enumerate(lines[:height]):
body.append(
f'<text x="{pad}" y="{pad + (idx + 1) * cell_h}" xml:space="preserve">{html.escape(line)}</text>'
)
return "\n".join(
[
f'<svg xmlns="http://www.w3.org/2000/svg" width="{svg_w}" height="{svg_h}" viewBox="0 0 {svg_w} {svg_h}">',
'<rect width="100%" height="100%" fill="#12161e"/>',
'<style>text{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:14px;fill:#d6deeb}</style>',
*body,
"</svg>",
]
)
body.append(f'<text x="{pad}" y="{pad + (idx + 1) * cell_h}" xml:space="preserve">{html.escape(line)}</text>')
return "\n".join([
f'<svg xmlns="http://www.w3.org/2000/svg" width="{svg_w}" height="{svg_h}" viewBox="0 0 {svg_w} {svg_h}">',
'<rect width="100%" height="100%" fill="#12161e"/>',
'<style>text{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:14px;fill:#d6deeb}</style>',
*body,
"</svg>",
])
def html_for_lines(lines: list[str], raw_svg_name: str) -> str:
def html_for_lines(lines: list[str], raw_svg_name: str, scenario: Scenario) -> str:
escaped = html.escape("\n".join(lines))
meta = html.escape(f"{scenario.id} {scenario.width}x{scenario.height} {scenario.file_state} {scenario.input_profile} {scenario.mode_surface} {scenario.color_mode}")
return f"""<!doctype html>
<meta charset="utf-8">
<title>mim terminal-e2e</title>
<title>mim terminal-e2e {html.escape(scenario.id)}</title>
<style>
body {{ margin: 0; background: #0f131a; color: #d6deeb; }}
.meta {{ color: #7ee787; font: 12px ui-monospace, monospace; padding: 8px 12px 0; }}
.terminal {{ white-space: pre; font: 14px/18px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; padding: 12px; }}
a {{ color: #7ee787; }}
</style>
<div class="meta">{meta}</div>
<div class="terminal">{escaped}</div>
<p><a href="{html.escape(raw_svg_name)}">SVG screenshot artifact</a></p>
"""
def write_artifacts(out_dir: Path, transcript: bytes, lines: list[str], width: int, height: int, saved: str | None) -> None:
def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines: list[str], saved: str | None) -> None:
visible = visible_controls(transcript)
normalized_visible = normalize_visible_controls(visible)
out_dir.mkdir(parents=True, exist_ok=True)
@@ -210,11 +233,15 @@ def write_artifacts(out_dir: Path, transcript: bytes, lines: list[str], width: i
(out_dir / "visible-controls.txt").write_text(visible, encoding="utf-8")
(out_dir / "visible-controls.normalized.txt").write_text(normalized_visible, encoding="utf-8")
(out_dir / "transcript.txt").write_text("\n".join(lines) + "\n", encoding="utf-8")
(out_dir / "screenshot.svg").write_text(svg_for_lines(lines, width, height), encoding="utf-8")
(out_dir / "terminal.html").write_text(html_for_lines(lines, "screenshot.svg"), encoding="utf-8")
(out_dir / "screenshot.svg").write_text(svg_for_lines(lines, scenario.width, scenario.height), encoding="utf-8")
(out_dir / "terminal.html").write_text(html_for_lines(lines, "screenshot.svg", scenario), encoding="utf-8")
snapshot = [
f"scenario\t{SCENARIO_ID}",
f"viewport\t{width}x{height}",
f"scenario\t{scenario.id}",
f"viewport\t{scenario.width}x{scenario.height}",
f"file_state\t{scenario.file_state}",
f"input_profile\t{scenario.input_profile}",
f"mode_surface\t{scenario.mode_surface}",
f"color_mode\t{scenario.color_mode}",
f"raw_sha256\t{hashlib.sha256(transcript).hexdigest()}",
f"bare_lf\t{str(has_bare_lf(transcript)).lower()}",
f"saved_sha256\t{hashlib.sha256((saved or '').encode('utf-8')).hexdigest()}",
@@ -223,19 +250,67 @@ def write_artifacts(out_dir: Path, transcript: bytes, lines: list[str], width: i
(out_dir / "manifest.tsv").write_text("key\tvalue\n" + "\n".join(snapshot) + "\n", encoding="utf-8")
def run_scenario(mim: Path, out_dir: Path, width: int, height: int) -> None:
self_test()
if out_dir.exists():
shutil.rmtree(out_dir)
out_dir.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="mim-terminal-e2e-") as tmp:
target = Path(tmp) / "note.txt"
target.write_text("", encoding="utf-8")
pid, fd = spawn_under_pty([str(mim), str(target)], width, height)
def setup_empty(tmp: Path) -> tuple[Path, str | None]:
target = tmp / "empty.txt"
target.write_text("", encoding="utf-8")
return target, ""
def setup_existing(tmp: Path) -> tuple[Path, str | None]:
target = tmp / "existing.txt"
target.write_text("alpha\nbeta\n", encoding="utf-8")
return target, "alpha\nbeta\n"
def setup_new(tmp: Path) -> tuple[Path, str | None]:
return tmp / "new-file.txt", ""
def setup_directory(tmp: Path) -> tuple[Path, str | None]:
root = tmp / "repo"
root.mkdir()
(root / "one.zig").write_text("pub fn main() void {}\n", encoding="utf-8")
return root, None
MOBILE_SAVE_QUIT = tuple(bytes([b]) for b in b"what is up n w q")
ATTACHED_SAVE_QUIT = (b"\x1b[F", b"!", b" ", b"n", b" ", b"w", b" ", b"q")
PANEL_QUIT = (b" ", b"n", b" ", b"q")
IOS_DEFAULT_KEYS = set(b"abcdefghijklmnopqrstuvwxyz ")
SCENARIOS = [
Scenario("raw-key-chrome-crlf", 48, 16, "empty", "ios-default-qwertz-space-path", "insert-normal", "truecolor", "file", setup_empty, MOBILE_SAVE_QUIT, "what is up", ("1│", "▌", "mode:normal")),
Scenario("attached-existing-medium", 72, 18, "existing", "attached-keyboard", "insert-normal", "truecolor", "file", setup_existing, ATTACHED_SAVE_QUIT, "alpha!\nbeta", ("1│", "2│", "▌")),
Scenario("new-file-mono-narrow", 40, 12, "new", "ios-default-qwertz-space-path", "insert-normal", "mono", "file", setup_new, MOBILE_SAVE_QUIT, "what is up", ("1│", "▌", "mode:normal")),
Scenario("directory-panel-narrow", 52, 12, "directory", "ios-default-qwertz-space-path", "panel", "mono", "directory", setup_directory, PANEL_QUIT, None, ("file", "one.zig")),
]
def assert_input_profile_is_ergonomic(scenario: Scenario) -> None:
if scenario.input_profile != "ios-default-qwertz-space-path":
return
for payload in scenario.payloads:
for byte in payload:
if byte not in IOS_DEFAULT_KEYS:
raise SystemExit(
f"{scenario.id}: iOS default keyboard scenario uses non-default key byte 0x{byte:02x}; "
"move it to an attached-keyboard scenario or redesign the mobile path"
)
def run_one(mim: Path, root_out: Path, scenario: Scenario) -> tuple[str, Path]:
assert_input_profile_is_ergonomic(scenario)
out_dir = root_out / scenario.id
with tempfile.TemporaryDirectory(prefix="mim-terminal-e2e-") as tmp_name:
tmp = Path(tmp_name)
target, expected_initial = scenario.setup(tmp)
_ = expected_initial
pid, fd = spawn_under_pty([str(mim), target.name], scenario.width, scenario.height, scenario.color_mode, tmp)
transcript = bytearray()
grid = TerminalGrid(width, height)
grid = TerminalGrid(scenario.width, scenario.height)
try:
for payload in [b"", b"w", b"h", b"a", b"t", b" ", b"i", b"s", b" ", b"u", b"p", b" ", b"n", b" ", b"w", b" ", b"q"]:
for payload in (b"", *scenario.payloads):
if payload:
os.write(fd, payload)
chunk = read_available(fd)
@@ -249,50 +324,75 @@ def run_scenario(mim: Path, out_dir: Path, width: int, height: int) -> None:
grid.feed(chunk)
if done_pid == pid:
if status != 0:
raise SystemExit(f"mim exited non-zero: status={status}")
write_artifacts(out_dir, scenario, bytes(transcript), grid.lines(), None)
raise SystemExit(f"{scenario.id}: mim exited non-zero: status={status}; artifacts={out_dir}")
break
else:
write_artifacts(out_dir, bytes(transcript), grid.lines(), width, height, None)
write_artifacts(out_dir, scenario, bytes(transcript), grid.lines(), None)
os.kill(pid, signal.SIGTERM)
raise SystemExit(f"mim did not exit after save+quit script; artifacts={out_dir}")
raise SystemExit(f"{scenario.id}: mim did not exit after script; artifacts={out_dir}")
finally:
try:
os.close(fd)
except OSError:
pass
saved = target.read_text(encoding="utf-8")
saved: str | None = None
if scenario.target_kind == "file" and target.exists():
saved = target.read_text(encoding="utf-8")
lines = grid.lines()
raw = bytes(transcript)
write_artifacts(out_dir, raw, lines, width, height, saved)
write_artifacts(out_dir, scenario, raw, lines, saved)
failures: list[str] = []
if saved != "what is up":
failures.append(f"saved file mismatch: {saved!r}")
joined = "\n".join(lines)
raw_text = raw.decode("utf-8", errors="ignore")
if scenario.expect_saved is not None and saved != scenario.expect_saved:
failures.append(f"saved file mismatch: {saved!r} != {scenario.expect_saved!r}")
if has_bare_lf(raw):
failures.append("raw terminal output contains bare LF; expected CRLF in raw mode")
if "1│" not in raw_text:
failures.append("line-number gutter missing from terminal output")
if "▌" not in raw_text:
failures.append("cursor marker missing from terminal output")
if "mode:normal" not in raw_text and "saved" not in raw_text:
failures.append("expected editor status not observed")
if "^[" in raw_text:
failures.append("escape appears echoed as ^[")
if "mode:normal" not in raw_text:
failures.append("Space-n did not leave insert mode before save/quit")
if any(len(line) > width for line in lines):
plain_text = strip_csi(raw_text)
for required in scenario.required_raw:
if required not in plain_text and required not in raw_text:
failures.append(f"required terminal text missing: {required!r}")
if scenario.color_mode == "truecolor" and "\x1b[38;2;" not in raw_text:
failures.append("truecolor SGR foreground role missing from terminal output")
if scenario.expect_saved and scenario.expect_saved in plain_text and f"▌{scenario.expect_saved}" in plain_text:
failures.append("cursor marker is pinned before typed content instead of following cursor column")
if scenario.expect_saved and scenario.expect_saved.replace("\n", "") in plain_text:
compact_saved = scenario.expect_saved.replace("\n", "")
if f"▌{compact_saved}" in plain_text:
failures.append("cursor marker is pinned before typed content instead of following cursor column")
for forbidden in scenario.forbidden_raw:
if forbidden in raw_text:
failures.append(f"forbidden terminal text present: {forbidden!r}")
if scenario.mode_surface != "panel" and "mode:normal" not in raw_text:
failures.append("scenario never reached normal mode")
if any(len(line) > scenario.width for line in lines):
failures.append("terminal line exceeds viewport width")
if failures:
print(f"artifacts\t{out_dir}", file=sys.stderr)
for failure in failures:
print(f"FAIL\t{failure}", file=sys.stderr)
print(f"FAIL\t{scenario.id}\t{failure}", file=sys.stderr)
raise SystemExit(1)
return scenario.id, out_dir
print("scenario\tstatus\tartifacts")
print(f"raw-key-chrome-crlf\tPASS\t{out_dir}")
def run_matrix(mim: Path, out_root: Path, only: set[str] | None) -> None:
self_test()
selected = [scenario for scenario in SCENARIOS if only is None or scenario.id in only]
unknown = (only or set()) - {scenario.id for scenario in SCENARIOS}
if unknown:
raise SystemExit(f"unknown scenario id(s): {', '.join(sorted(unknown))}")
if out_root.exists():
shutil.rmtree(out_root)
out_root.mkdir(parents=True, exist_ok=True)
rows = ["scenario\tstatus\tartifacts"]
matrix_rows = ["scenario\tviewport\tfile_state\tinput_profile\tmode_surface\tcolor_mode\tartifacts"]
for scenario in selected:
scenario_id, artifact_dir = run_one(mim, out_root, scenario)
rows.append(f"{scenario_id}\tPASS\t{artifact_dir}")
matrix_rows.append(f"{scenario.id}\t{scenario.width}x{scenario.height}\t{scenario.file_state}\t{scenario.input_profile}\t{scenario.mode_surface}\t{scenario.color_mode}\t{artifact_dir}")
(out_root / "matrix.tsv").write_text("\n".join(matrix_rows) + "\n", encoding="utf-8")
print("\n".join(rows))
def main() -> int:
@@ -300,17 +400,16 @@ def main() -> int:
description="Run mim through a PTY and write browser-viewable terminal E2E artifacts.",
epilog=(
"Golden policy: investigate before updating artifacts. Semantic snapshots "
"(manifest.tsv, transcript.txt, visible-controls.normalized.txt) are the "
"(matrix.tsv, manifest.tsv, transcript.txt, visible-controls.normalized.txt) are the "
"primary review surface; screenshot.svg/terminal.html are visual evidence. "
"It is safe to update artifacts only after explaining intentional UI/control-byte changes."
),
)
parser.add_argument("mim", type=Path, help="path to compiled mim binary")
parser.add_argument("--out", type=Path, default=Path(f".zig-cache/terminal-e2e/{SCENARIO_ID}"))
parser.add_argument("--width", type=int, default=48)
parser.add_argument("--height", type=int, default=16)
parser.add_argument("--out", type=Path, default=Path(".zig-cache/terminal-e2e"))
parser.add_argument("--scenario", action="append", help="run only this scenario id; may be repeated")
args = parser.parse_args()
run_scenario(args.mim.resolve(), args.out, args.width, args.height)
run_matrix(args.mim.resolve(), args.out, set(args.scenario) if args.scenario else None)
return 0