Assert terminal E2E chrome roles
This commit is contained in:
+10
-5
@@ -15,20 +15,25 @@ Matrix input profiles:
|
|||||||
|
|
||||||
Artifact roles:
|
Artifact roles:
|
||||||
|
|
||||||
- `manifest.tsv`: stable scenario facts, hashes, bare-LF status, and artifact list.
|
- `manifest.tsv`: stable scenario facts, hashes, bare-LF status, seen ANSI roles,
|
||||||
|
and artifact list.
|
||||||
- `visible-controls.normalized.txt`: primary semantic failure artifact. It shows
|
- `visible-controls.normalized.txt`: primary semantic failure artifact. It shows
|
||||||
CR as `␍`, LF as `␊`, ESC as `␛`, tabs as `⇥`, and normalizes volatile temp
|
CR as `␍`, LF as `␊`, ESC as `␛`, tabs as `⇥`, and normalizes volatile temp
|
||||||
paths.
|
paths.
|
||||||
- `transcript.txt`: semantic terminal grid after escape/control interpretation.
|
- `transcript.txt`: semantic terminal grid after escape/control interpretation,
|
||||||
- `screenshot.svg` and `terminal.html`: browser-viewable visual evidence.
|
including debug-only role markers such as `☻` for a cursor cell and `░` for
|
||||||
|
selection.
|
||||||
|
- `screenshot.svg` and `terminal.html`: browser-viewable visual evidence. SVG
|
||||||
|
includes color swatches for seen cursor/current-line/status/selection roles.
|
||||||
|
- `widths.tsv`: per-line semantic width report used to catch viewport overflow.
|
||||||
- `raw.bin`: exact PTY bytes for low-level debugging.
|
- `raw.bin`: exact PTY bytes for low-level debugging.
|
||||||
|
|
||||||
Golden policy:
|
Golden policy:
|
||||||
|
|
||||||
- Investigate before updating any expected artifact or assertion.
|
- Investigate before updating any expected artifact or assertion.
|
||||||
- Prefer semantic artifacts (`manifest.tsv`, `visible-controls.normalized.txt`,
|
- Prefer semantic artifacts (`manifest.tsv`, `visible-controls.normalized.txt`,
|
||||||
`transcript.txt`) for assertions; visual artifacts explain what a user would
|
`transcript.txt`, `widths.tsv`) for assertions; visual artifacts explain what a
|
||||||
see.
|
user would see.
|
||||||
- Updating artifacts is safe only after the change is intentional and the receipt
|
- Updating artifacts is safe only after the change is intentional and the receipt
|
||||||
names the changed UI/control-byte behavior.
|
names the changed UI/control-byte behavior.
|
||||||
- Adding a mobile scenario must preserve the `ios-default-qwertz-space-path`
|
- Adding a mobile scenario must preserve the `ios-default-qwertz-space-path`
|
||||||
|
|||||||
+34
-8
@@ -42,6 +42,7 @@ class Scenario:
|
|||||||
expect_saved: str | None
|
expect_saved: str | None
|
||||||
required_raw: tuple[str, ...]
|
required_raw: tuple[str, ...]
|
||||||
forbidden_raw: tuple[str, ...] = ("^[",)
|
forbidden_raw: tuple[str, ...] = ("^[",)
|
||||||
|
required_attrs: tuple[str, ...] = ("cursor", "status", "current-line")
|
||||||
|
|
||||||
|
|
||||||
def visible_controls(data: bytes) -> str:
|
def visible_controls(data: bytes) -> str:
|
||||||
@@ -295,10 +296,14 @@ def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines:
|
|||||||
visible = visible_controls(transcript)
|
visible = visible_controls(transcript)
|
||||||
normalized_visible = normalize_visible_controls(visible)
|
normalized_visible = normalize_visible_controls(visible)
|
||||||
raw_seen_attrs = set(seen_attrs)
|
raw_seen_attrs = set(seen_attrs)
|
||||||
if "48;2;245;197;92" in normalized_visible:
|
for sgr, attr in (
|
||||||
raw_seen_attrs.add("cursor")
|
("48;2;245;197;92", "cursor"),
|
||||||
if "48;2;64;96;140" in normalized_visible:
|
("48;2;64;96;140", "selection"),
|
||||||
raw_seen_attrs.add("selection")
|
("48;2;26;31;43", "current-line"),
|
||||||
|
("48;2;126;231;135", "status"),
|
||||||
|
):
|
||||||
|
if sgr in normalized_visible:
|
||||||
|
raw_seen_attrs.add(attr)
|
||||||
if "cursor" in raw_seen_attrs and not any("☻" in line for line in lines):
|
if "cursor" in raw_seen_attrs and not any("☻" in line for line in lines):
|
||||||
lines = [*lines, "attr:cursor ☻"]
|
lines = [*lines, "attr:cursor ☻"]
|
||||||
if "selection" in raw_seen_attrs and not any("░" in line for line in lines):
|
if "selection" in raw_seen_attrs and not any("░" in line for line in lines):
|
||||||
@@ -310,6 +315,10 @@ def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines:
|
|||||||
(out_dir / "transcript.txt").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
(out_dir / "transcript.txt").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||||
(out_dir / "screenshot.svg").write_text(svg_for_cells(rows, scenario.width, scenario.height, raw_seen_attrs), encoding="utf-8")
|
(out_dir / "screenshot.svg").write_text(svg_for_cells(rows, scenario.width, scenario.height, raw_seen_attrs), encoding="utf-8")
|
||||||
(out_dir / "terminal.html").write_text(html_for_lines(lines, "screenshot.svg", scenario), encoding="utf-8")
|
(out_dir / "terminal.html").write_text(html_for_lines(lines, "screenshot.svg", scenario), encoding="utf-8")
|
||||||
|
width_report = ["line width text"]
|
||||||
|
for idx, line in enumerate(lines, start=1):
|
||||||
|
width_report.append(f"{idx} {len(line)} {line}")
|
||||||
|
(out_dir / "widths.tsv").write_text("\n".join(width_report) + "\n", encoding="utf-8")
|
||||||
snapshot = [
|
snapshot = [
|
||||||
f"scenario\t{scenario.id}",
|
f"scenario\t{scenario.id}",
|
||||||
f"viewport\t{scenario.width}x{scenario.height}",
|
f"viewport\t{scenario.width}x{scenario.height}",
|
||||||
@@ -320,7 +329,8 @@ def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines:
|
|||||||
f"raw_sha256\t{hashlib.sha256(transcript).hexdigest()}",
|
f"raw_sha256\t{hashlib.sha256(transcript).hexdigest()}",
|
||||||
f"bare_lf\t{str(has_bare_lf(transcript)).lower()}",
|
f"bare_lf\t{str(has_bare_lf(transcript)).lower()}",
|
||||||
f"saved_sha256\t{hashlib.sha256((saved or '').encode('utf-8')).hexdigest()}",
|
f"saved_sha256\t{hashlib.sha256((saved or '').encode('utf-8')).hexdigest()}",
|
||||||
"artifacts\traw.bin visible-controls.txt visible-controls.normalized.txt transcript.txt screenshot.svg terminal.html manifest.tsv",
|
f"attrs_seen\t{','.join(sorted(raw_seen_attrs))}",
|
||||||
|
"artifacts\traw.bin visible-controls.txt visible-controls.normalized.txt transcript.txt screenshot.svg terminal.html widths.tsv manifest.tsv",
|
||||||
]
|
]
|
||||||
(out_dir / "manifest.tsv").write_text("key\tvalue\n" + "\n".join(snapshot) + "\n", encoding="utf-8")
|
(out_dir / "manifest.tsv").write_text("key\tvalue\n" + "\n".join(snapshot) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
@@ -400,7 +410,7 @@ SCENARIOS = [
|
|||||||
Scenario("percent-match-jump", 56, 12, "brackets", "attached-keyboard", "match-jump", "truecolor", "file", setup_brackets, (b"%", b"i", b"X", b"\x1b", b" ", b"w", b" ", b"q"), "(abX)", ("1│", "☻")),
|
Scenario("percent-match-jump", 56, 12, "brackets", "attached-keyboard", "match-jump", "truecolor", "file", setup_brackets, (b"%", b"i", b"X", b"\x1b", b" ", b"w", b" ", b"q"), "(abX)", ("1│", "☻")),
|
||||||
Scenario("select-mode-colors", 56, 12, "short", "attached-keyboard", "select", "truecolor", "file", setup_short, (b"s", b"w", b"n", b" ", b"q"), "abcdef\n", ("attr:selection", "░", "mode:normal")),
|
Scenario("select-mode-colors", 56, 12, "short", "attached-keyboard", "select", "truecolor", "file", setup_short, (b"s", b"w", b"n", b" ", b"q"), "abcdef\n", ("attr:selection", "░", "mode:normal")),
|
||||||
Scenario("dirty-discard-shift-q", 52, 12, "empty", "ios-default-qwertz-space-path", "dirty-discard", "truecolor", "file", setup_empty, tuple(bytes([b]) for b in b" ps Q"), "", ("1│", "☻")),
|
Scenario("dirty-discard-shift-q", 52, 12, "empty", "ios-default-qwertz-space-path", "dirty-discard", "truecolor", "file", setup_empty, tuple(bytes([b]) for b in b" ps Q"), "", ("1│", "☻")),
|
||||||
Scenario("directory-panel-narrow", 52, 12, "directory", "ios-default-qwertz-space-path", "panel", "mono", "directory", setup_directory, PANEL_QUIT, None, ("file", "one.zig")),
|
Scenario("directory-panel-narrow", 52, 12, "directory", "ios-default-qwertz-space-path", "panel", "mono", "directory", setup_directory, PANEL_QUIT, None, ("file", "one.zig"), required_attrs=()),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -467,15 +477,31 @@ def run_one(mim: Path, root_out: Path, scenario: Scenario) -> tuple[str, Path]:
|
|||||||
if has_bare_lf(raw):
|
if has_bare_lf(raw):
|
||||||
failures.append("raw terminal output contains bare LF; expected CRLF in raw mode")
|
failures.append("raw terminal output contains bare LF; expected CRLF in raw mode")
|
||||||
plain_text = strip_csi(raw_text)
|
plain_text = strip_csi(raw_text)
|
||||||
if "48;2;245;197;92" in raw_text:
|
raw_attrs: set[str] = set()
|
||||||
|
for sgr, attr in (
|
||||||
|
("48;2;245;197;92", "cursor"),
|
||||||
|
("48;2;64;96;140", "selection"),
|
||||||
|
("48;2;26;31;43", "current-line"),
|
||||||
|
("48;2;126;231;135", "status"),
|
||||||
|
):
|
||||||
|
if sgr in raw_text:
|
||||||
|
raw_attrs.add(attr)
|
||||||
|
if "cursor" in raw_attrs:
|
||||||
plain_text += "\nattr:cursor ☻"
|
plain_text += "\nattr:cursor ☻"
|
||||||
if "48;2;64;96;140" in raw_text:
|
if "selection" in raw_attrs:
|
||||||
plain_text += "\nattr:selection ░"
|
plain_text += "\nattr:selection ░"
|
||||||
for required in scenario.required_raw:
|
for required in scenario.required_raw:
|
||||||
if required not in plain_text and required not in raw_text:
|
if required not in plain_text and required not in raw_text:
|
||||||
failures.append(f"required terminal text missing: {required!r}")
|
failures.append(f"required terminal text missing: {required!r}")
|
||||||
|
for attr in scenario.required_attrs:
|
||||||
|
if attr not in raw_attrs:
|
||||||
|
failures.append(f"required ANSI role missing: {attr}")
|
||||||
if scenario.color_mode == "truecolor" and "\x1b[38;2;" not in raw_text:
|
if scenario.color_mode == "truecolor" and "\x1b[38;2;" not in raw_text:
|
||||||
failures.append("truecolor SGR foreground role missing from terminal output")
|
failures.append("truecolor SGR foreground role missing from terminal output")
|
||||||
|
if scenario.color_mode == "mono" and scenario.mode_surface != "panel":
|
||||||
|
for text in ("mode:", " 1│"):
|
||||||
|
if text not in plain_text:
|
||||||
|
failures.append(f"mono/degraded transcript lost legible chrome text: {text!r}")
|
||||||
if "▌" in plain_text:
|
if "▌" in plain_text:
|
||||||
failures.append("terminal output contains layout-changing cursor glyph; cursor must be a cell attribute")
|
failures.append("terminal output contains layout-changing cursor glyph; cursor must be a cell attribute")
|
||||||
for forbidden in scenario.forbidden_raw:
|
for forbidden in scenario.forbidden_raw:
|
||||||
|
|||||||
Reference in New Issue
Block a user