diff --git a/tools/TERMINAL_E2E.md b/tools/TERMINAL_E2E.md index aad008a..5f85fe1 100644 --- a/tools/TERMINAL_E2E.md +++ b/tools/TERMINAL_E2E.md @@ -26,14 +26,17 @@ Artifact roles: - `screenshot.svg` and `terminal.html`: browser-viewable visual evidence. SVG includes color swatches for seen cursor/current-line/status/selection roles. - `widths.tsv`: per-line semantic width report used to catch viewport overflow. +- `keystrokes.tsv`: logical user key-event count and raw byte count for scenario + payloads. A CSI sequence such as PageDown counts as one attached-key event + while still recording all raw bytes sent. - `raw.bin`: exact PTY bytes for low-level debugging. Golden policy: - Investigate before updating any expected artifact or assertion. - Prefer semantic artifacts (`manifest.tsv`, `visible-controls.normalized.txt`, - `transcript.txt`, `widths.tsv`) for assertions; visual artifacts explain what a - user would see. + `transcript.txt`, `widths.tsv`, `keystrokes.tsv`) for assertions; visual + artifacts explain what a user would see. - Updating artifacts is safe only after the change is intentional and the receipt names the changed UI/control-byte behavior. - Adding a mobile scenario must preserve the `ios-default-qwertz-space-path` @@ -41,5 +44,8 @@ Golden policy: - Bare LF in raw-mode terminal output is a regression unless a scenario explicitly proves a non-raw stream. Raw-mode rendering should use CRLF to avoid diagonal terminal skew. +- Scenario `max_key_events` budgets are regression guards for workflow burden, + not speed benchmarks. Raise a budget only after explaining the intentional + workflow change in the receipt. - Generated artifacts must remain under ignored build output and must not dirty a clean source tree. diff --git a/tools/terminal_e2e.py b/tools/terminal_e2e.py index 9e99e64..0ac38f9 100755 --- a/tools/terminal_e2e.py +++ b/tools/terminal_e2e.py @@ -43,6 +43,7 @@ class Scenario: required_raw: tuple[str, ...] forbidden_raw: tuple[str, ...] = ("^[",) required_attrs: tuple[str, ...] = ("cursor", "status", "current-line") + max_key_events: int | None = None def visible_controls(data: bytes) -> str: @@ -292,6 +293,47 @@ a {{ color: #7ee787; }} """ +def payload_label(payload: bytes) -> str: + if payload == b" ": + return "Space" + if payload == b"\x1b": + return "Esc" + if payload == b"\r": + return "Enter" + if payload == b"\t": + return "Tab" + names = { + b"\x1b[A": "ArrowUp", + b"\x1b[B": "ArrowDown", + b"\x1b[C": "ArrowRight", + b"\x1b[D": "ArrowLeft", + b"\x1b[F": "End", + b"\x1b[H": "Home", + b"\x1b[5~": "PageUp", + b"\x1b[6~": "PageDown", + } + if payload in names: + return names[payload] + try: + text = payload.decode("utf-8") + except UnicodeDecodeError: + return "bytes" + if text.isprintable(): + return text + return "bytes" + + +def keystroke_rows(scenario: Scenario) -> tuple[list[str], int, int]: + rows = ["step\tkey_event\traw_bytes\tbytes_hex\tutf8"] + raw_bytes = 0 + for idx, payload in enumerate(scenario.payloads, start=1): + raw_bytes += len(payload) + utf8 = payload.decode("utf-8", errors="replace") + visible = visible_controls(payload).replace("\n", "\\n") + rows.append(f"{idx}\t{payload_label(payload)}\t{len(payload)}\t{payload.hex()}\t{visible or utf8}") + return rows, len(scenario.payloads), raw_bytes + + def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines: list[str], rows: list[list[Cell]], seen_attrs: set[str], saved: str | None) -> None: visible = visible_controls(transcript) normalized_visible = normalize_visible_controls(visible) @@ -308,6 +350,10 @@ def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines: lines = [*lines, "attr:cursor ☻"] if "selection" in raw_seen_attrs and not any("░" in line for line in lines): lines = [*lines, "attr:selection ░"] + key_rows, key_events, raw_input_bytes = keystroke_rows(scenario) + budget_status = "unbudgeted" + if scenario.max_key_events is not None: + budget_status = "pass" if key_events <= scenario.max_key_events else "fail" out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "raw.bin").write_bytes(transcript) (out_dir / "visible-controls.txt").write_text(visible, encoding="utf-8") @@ -319,6 +365,7 @@ def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines: for idx, line in enumerate(lines, start=1): width_report.append(f"{idx} {len(line)} {line}") (out_dir / "widths.tsv").write_text("\n".join(width_report) + "\n", encoding="utf-8") + (out_dir / "keystrokes.tsv").write_text("\n".join(key_rows) + "\n", encoding="utf-8") snapshot = [ f"scenario\t{scenario.id}", f"viewport\t{scenario.width}x{scenario.height}", @@ -330,7 +377,11 @@ def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines: f"bare_lf\t{str(has_bare_lf(transcript)).lower()}", f"saved_sha256\t{hashlib.sha256((saved or '').encode('utf-8')).hexdigest()}", f"attrs_seen\t{','.join(sorted(raw_seen_attrs))}", - "artifacts\traw.bin visible-controls.txt visible-controls.normalized.txt transcript.txt screenshot.svg terminal.html widths.tsv manifest.tsv", + f"key_events\t{key_events}", + f"raw_input_bytes\t{raw_input_bytes}", + f"max_key_events\t{scenario.max_key_events if scenario.max_key_events is not None else ''}", + f"key_budget\t{budget_status}", + "artifacts\traw.bin visible-controls.txt visible-controls.normalized.txt transcript.txt screenshot.svg terminal.html widths.tsv keystrokes.tsv manifest.tsv", ] (out_dir / "manifest.tsv").write_text("key\tvalue\n" + "\n".join(snapshot) + "\n", encoding="utf-8") @@ -396,9 +447,9 @@ PANEL_QUIT = (b" ", b"q") IOS_DEFAULT_KEYS = set(b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ") SCENARIOS = [ - Scenario("raw-key-chrome-crlf", 48, 16, "empty", "ios-default-qwertz-space-path", "normal-symbol-save", "truecolor", "file", setup_empty, MOBILE_SYMBOL_SAVE_QUIT, "/", ("1│", "☻", "mode:normal")), + Scenario("raw-key-chrome-crlf", 48, 16, "empty", "ios-default-qwertz-space-path", "normal-symbol-save", "truecolor", "file", setup_empty, MOBILE_SYMBOL_SAVE_QUIT, "/", ("1│", "☻", "mode:normal"), max_key_events=7), Scenario("ios-normal-replace-shift-path", 44, 12, "existing", "ios-default-qwertz-space-path", "normal-replace", "truecolor", "file", setup_existing, MOBILE_REPLACE_SAVE_QUIT, "xlpha\nbeta", ("1│", "☻", "mode:normal")), - Scenario("attached-existing-medium", 72, 18, "existing", "attached-keyboard", "insert-normal", "truecolor", "file", setup_existing, ATTACHED_SAVE_QUIT, "alpha!\nbeta", ("1│", "2│", "☻")), + Scenario("attached-existing-medium", 72, 18, "existing", "attached-keyboard", "insert-normal", "truecolor", "file", setup_existing, ATTACHED_SAVE_QUIT, "alpha!\nbeta", ("1│", "2│", "☻"), max_key_events=8), Scenario("new-file-mono-narrow", 40, 12, "new", "ios-default-qwertz-space-path", "normal-symbol-save", "mono", "file", setup_new, MOBILE_SYMBOL_SAVE_QUIT, "/", ("1│", "☻", "mode:normal")), Scenario("esc-attached-key-mode-switch", 56, 14, "empty", "attached-keyboard", "insert-normal", "truecolor", "file", setup_empty, (b"i", b"e", b"s", b"c", b"\x1b", b" ", b"w", b" ", b"q"), "esc", ("1│", "☻", "mode:normal")), Scenario("attached-cursor-left-insert", 56, 14, "empty", "attached-keyboard", "insert-cursor", "truecolor", "file", setup_empty, (b"i", b"a", b"b", b"\x1b[D", b"X", b"\x1b", b" ", b"w", b" ", b"q"), "aXb", ("1│", "☻", "aX", "mode:normal")), @@ -472,6 +523,9 @@ def run_one(mim: Path, root_out: Path, scenario: Scenario) -> tuple[str, Path]: write_artifacts(out_dir, scenario, raw, lines, grid.colored_rows(), grid.attrs_seen(), saved) failures: list[str] = [] raw_text = raw.decode("utf-8", errors="ignore") + _, key_events, _ = keystroke_rows(scenario) + if scenario.max_key_events is not None and key_events > scenario.max_key_events: + failures.append(f"key budget exceeded: {key_events} > {scenario.max_key_events}") if scenario.expect_saved is not None and saved != scenario.expect_saved: failures.append(f"saved file mismatch: {saved!r} != {scenario.expect_saved!r}") if has_bare_lf(raw): @@ -529,11 +583,12 @@ def run_matrix(mim: Path, out_root: Path, only: set[str] | None) -> None: shutil.rmtree(out_root) out_root.mkdir(parents=True, exist_ok=True) rows = ["scenario\tstatus\tartifacts"] - matrix_rows = ["scenario\tviewport\tfile_state\tinput_profile\tmode_surface\tcolor_mode\tartifacts"] + matrix_rows = ["scenario\tviewport\tfile_state\tinput_profile\tmode_surface\tcolor_mode\tkey_events\tmax_key_events\tartifacts"] for scenario in selected: scenario_id, artifact_dir = run_one(mim, out_root, scenario) rows.append(f"{scenario_id}\tPASS\t{artifact_dir}") - matrix_rows.append(f"{scenario.id}\t{scenario.width}x{scenario.height}\t{scenario.file_state}\t{scenario.input_profile}\t{scenario.mode_surface}\t{scenario.color_mode}\t{artifact_dir}") + _, key_events, _ = keystroke_rows(scenario) + matrix_rows.append(f"{scenario.id}\t{scenario.width}x{scenario.height}\t{scenario.file_state}\t{scenario.input_profile}\t{scenario.mode_surface}\t{scenario.color_mode}\t{key_events}\t{scenario.max_key_events if scenario.max_key_events is not None else ''}\t{artifact_dir}") (out_root / "matrix.tsv").write_text("\n".join(matrix_rows) + "\n", encoding="utf-8") print("\n".join(rows))