Add terminal E2E keystroke budgets
This commit is contained in:
slhx agent
@@ -26,14 +26,17 @@ Artifact roles:
|
||||
- `screenshot.svg` and `terminal.html`: browser-viewable visual evidence. SVG
|
||||
includes color swatches for seen cursor/current-line/status/selection roles.
|
||||
- `widths.tsv`: per-line semantic width report used to catch viewport overflow.
|
||||
- `keystrokes.tsv`: logical user key-event count and raw byte count for scenario
|
||||
payloads. A CSI sequence such as PageDown counts as one attached-key event
|
||||
while still recording all raw bytes sent.
|
||||
- `raw.bin`: exact PTY bytes for low-level debugging.
|
||||
|
||||
Golden policy:
|
||||
|
||||
- Investigate before updating any expected artifact or assertion.
|
||||
- Prefer semantic artifacts (`manifest.tsv`, `visible-controls.normalized.txt`,
|
||||
`transcript.txt`, `widths.tsv`) for assertions; visual artifacts explain what a
|
||||
user would see.
|
||||
`transcript.txt`, `widths.tsv`, `keystrokes.tsv`) for assertions; visual
|
||||
artifacts explain what a user would see.
|
||||
- Updating artifacts is safe only after the change is intentional and the receipt
|
||||
names the changed UI/control-byte behavior.
|
||||
- Adding a mobile scenario must preserve the `ios-default-qwertz-space-path`
|
||||
@@ -41,5 +44,8 @@ Golden policy:
|
||||
- Bare LF in raw-mode terminal output is a regression unless a scenario explicitly
|
||||
proves a non-raw stream. Raw-mode rendering should use CRLF to avoid diagonal
|
||||
terminal skew.
|
||||
- Scenario `max_key_events` budgets are regression guards for workflow burden,
|
||||
not speed benchmarks. Raise a budget only after explaining the intentional
|
||||
workflow change in the receipt.
|
||||
- Generated artifacts must remain under ignored build output and must not dirty a
|
||||
clean source tree.
|
||||
|
||||
+60
-5
@@ -43,6 +43,7 @@ class Scenario:
|
||||
required_raw: tuple[str, ...]
|
||||
forbidden_raw: tuple[str, ...] = ("^[",)
|
||||
required_attrs: tuple[str, ...] = ("cursor", "status", "current-line")
|
||||
max_key_events: int | None = None
|
||||
|
||||
|
||||
def visible_controls(data: bytes) -> str:
|
||||
@@ -292,6 +293,47 @@ a {{ color: #7ee787; }}
|
||||
"""
|
||||
|
||||
|
||||
def payload_label(payload: bytes) -> str:
|
||||
if payload == b" ":
|
||||
return "Space"
|
||||
if payload == b"\x1b":
|
||||
return "Esc"
|
||||
if payload == b"\r":
|
||||
return "Enter"
|
||||
if payload == b"\t":
|
||||
return "Tab"
|
||||
names = {
|
||||
b"\x1b[A": "ArrowUp",
|
||||
b"\x1b[B": "ArrowDown",
|
||||
b"\x1b[C": "ArrowRight",
|
||||
b"\x1b[D": "ArrowLeft",
|
||||
b"\x1b[F": "End",
|
||||
b"\x1b[H": "Home",
|
||||
b"\x1b[5~": "PageUp",
|
||||
b"\x1b[6~": "PageDown",
|
||||
}
|
||||
if payload in names:
|
||||
return names[payload]
|
||||
try:
|
||||
text = payload.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return "bytes"
|
||||
if text.isprintable():
|
||||
return text
|
||||
return "bytes"
|
||||
|
||||
|
||||
def keystroke_rows(scenario: Scenario) -> tuple[list[str], int, int]:
|
||||
rows = ["step\tkey_event\traw_bytes\tbytes_hex\tutf8"]
|
||||
raw_bytes = 0
|
||||
for idx, payload in enumerate(scenario.payloads, start=1):
|
||||
raw_bytes += len(payload)
|
||||
utf8 = payload.decode("utf-8", errors="replace")
|
||||
visible = visible_controls(payload).replace("\n", "\\n")
|
||||
rows.append(f"{idx}\t{payload_label(payload)}\t{len(payload)}\t{payload.hex()}\t{visible or utf8}")
|
||||
return rows, len(scenario.payloads), raw_bytes
|
||||
|
||||
|
||||
def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines: list[str], rows: list[list[Cell]], seen_attrs: set[str], saved: str | None) -> None:
|
||||
visible = visible_controls(transcript)
|
||||
normalized_visible = normalize_visible_controls(visible)
|
||||
@@ -308,6 +350,10 @@ def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines:
|
||||
lines = [*lines, "attr:cursor ☻"]
|
||||
if "selection" in raw_seen_attrs and not any("░" in line for line in lines):
|
||||
lines = [*lines, "attr:selection ░"]
|
||||
key_rows, key_events, raw_input_bytes = keystroke_rows(scenario)
|
||||
budget_status = "unbudgeted"
|
||||
if scenario.max_key_events is not None:
|
||||
budget_status = "pass" if key_events <= scenario.max_key_events else "fail"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
(out_dir / "raw.bin").write_bytes(transcript)
|
||||
(out_dir / "visible-controls.txt").write_text(visible, encoding="utf-8")
|
||||
@@ -319,6 +365,7 @@ def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines:
|
||||
for idx, line in enumerate(lines, start=1):
|
||||
width_report.append(f"{idx} {len(line)} {line}")
|
||||
(out_dir / "widths.tsv").write_text("\n".join(width_report) + "\n", encoding="utf-8")
|
||||
(out_dir / "keystrokes.tsv").write_text("\n".join(key_rows) + "\n", encoding="utf-8")
|
||||
snapshot = [
|
||||
f"scenario\t{scenario.id}",
|
||||
f"viewport\t{scenario.width}x{scenario.height}",
|
||||
@@ -330,7 +377,11 @@ def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines:
|
||||
f"bare_lf\t{str(has_bare_lf(transcript)).lower()}",
|
||||
f"saved_sha256\t{hashlib.sha256((saved or '').encode('utf-8')).hexdigest()}",
|
||||
f"attrs_seen\t{','.join(sorted(raw_seen_attrs))}",
|
||||
"artifacts\traw.bin visible-controls.txt visible-controls.normalized.txt transcript.txt screenshot.svg terminal.html widths.tsv manifest.tsv",
|
||||
f"key_events\t{key_events}",
|
||||
f"raw_input_bytes\t{raw_input_bytes}",
|
||||
f"max_key_events\t{scenario.max_key_events if scenario.max_key_events is not None else ''}",
|
||||
f"key_budget\t{budget_status}",
|
||||
"artifacts\traw.bin visible-controls.txt visible-controls.normalized.txt transcript.txt screenshot.svg terminal.html widths.tsv keystrokes.tsv manifest.tsv",
|
||||
]
|
||||
(out_dir / "manifest.tsv").write_text("key\tvalue\n" + "\n".join(snapshot) + "\n", encoding="utf-8")
|
||||
|
||||
@@ -396,9 +447,9 @@ PANEL_QUIT = (b" ", b"q")
|
||||
IOS_DEFAULT_KEYS = set(b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ")
|
||||
|
||||
SCENARIOS = [
|
||||
Scenario("raw-key-chrome-crlf", 48, 16, "empty", "ios-default-qwertz-space-path", "normal-symbol-save", "truecolor", "file", setup_empty, MOBILE_SYMBOL_SAVE_QUIT, "/", ("1│", "☻", "mode:normal")),
|
||||
Scenario("raw-key-chrome-crlf", 48, 16, "empty", "ios-default-qwertz-space-path", "normal-symbol-save", "truecolor", "file", setup_empty, MOBILE_SYMBOL_SAVE_QUIT, "/", ("1│", "☻", "mode:normal"), max_key_events=7),
|
||||
Scenario("ios-normal-replace-shift-path", 44, 12, "existing", "ios-default-qwertz-space-path", "normal-replace", "truecolor", "file", setup_existing, MOBILE_REPLACE_SAVE_QUIT, "xlpha\nbeta", ("1│", "☻", "mode:normal")),
|
||||
Scenario("attached-existing-medium", 72, 18, "existing", "attached-keyboard", "insert-normal", "truecolor", "file", setup_existing, ATTACHED_SAVE_QUIT, "alpha!\nbeta", ("1│", "2│", "☻")),
|
||||
Scenario("attached-existing-medium", 72, 18, "existing", "attached-keyboard", "insert-normal", "truecolor", "file", setup_existing, ATTACHED_SAVE_QUIT, "alpha!\nbeta", ("1│", "2│", "☻"), max_key_events=8),
|
||||
Scenario("new-file-mono-narrow", 40, 12, "new", "ios-default-qwertz-space-path", "normal-symbol-save", "mono", "file", setup_new, MOBILE_SYMBOL_SAVE_QUIT, "/", ("1│", "☻", "mode:normal")),
|
||||
Scenario("esc-attached-key-mode-switch", 56, 14, "empty", "attached-keyboard", "insert-normal", "truecolor", "file", setup_empty, (b"i", b"e", b"s", b"c", b"\x1b", b" ", b"w", b" ", b"q"), "esc", ("1│", "☻", "mode:normal")),
|
||||
Scenario("attached-cursor-left-insert", 56, 14, "empty", "attached-keyboard", "insert-cursor", "truecolor", "file", setup_empty, (b"i", b"a", b"b", b"\x1b[D", b"X", b"\x1b", b" ", b"w", b" ", b"q"), "aXb", ("1│", "☻", "aX", "mode:normal")),
|
||||
@@ -472,6 +523,9 @@ def run_one(mim: Path, root_out: Path, scenario: Scenario) -> tuple[str, Path]:
|
||||
write_artifacts(out_dir, scenario, raw, lines, grid.colored_rows(), grid.attrs_seen(), saved)
|
||||
failures: list[str] = []
|
||||
raw_text = raw.decode("utf-8", errors="ignore")
|
||||
_, key_events, _ = keystroke_rows(scenario)
|
||||
if scenario.max_key_events is not None and key_events > scenario.max_key_events:
|
||||
failures.append(f"key budget exceeded: {key_events} > {scenario.max_key_events}")
|
||||
if scenario.expect_saved is not None and saved != scenario.expect_saved:
|
||||
failures.append(f"saved file mismatch: {saved!r} != {scenario.expect_saved!r}")
|
||||
if has_bare_lf(raw):
|
||||
@@ -529,11 +583,12 @@ def run_matrix(mim: Path, out_root: Path, only: set[str] | None) -> None:
|
||||
shutil.rmtree(out_root)
|
||||
out_root.mkdir(parents=True, exist_ok=True)
|
||||
rows = ["scenario\tstatus\tartifacts"]
|
||||
matrix_rows = ["scenario\tviewport\tfile_state\tinput_profile\tmode_surface\tcolor_mode\tartifacts"]
|
||||
matrix_rows = ["scenario\tviewport\tfile_state\tinput_profile\tmode_surface\tcolor_mode\tkey_events\tmax_key_events\tartifacts"]
|
||||
for scenario in selected:
|
||||
scenario_id, artifact_dir = run_one(mim, out_root, scenario)
|
||||
rows.append(f"{scenario_id}\tPASS\t{artifact_dir}")
|
||||
matrix_rows.append(f"{scenario.id}\t{scenario.width}x{scenario.height}\t{scenario.file_state}\t{scenario.input_profile}\t{scenario.mode_surface}\t{scenario.color_mode}\t{artifact_dir}")
|
||||
_, key_events, _ = keystroke_rows(scenario)
|
||||
matrix_rows.append(f"{scenario.id}\t{scenario.width}x{scenario.height}\t{scenario.file_state}\t{scenario.input_profile}\t{scenario.mode_surface}\t{scenario.color_mode}\t{key_events}\t{scenario.max_key_events if scenario.max_key_events is not None else ''}\t{artifact_dir}")
|
||||
(out_root / "matrix.tsv").write_text("\n".join(matrix_rows) + "\n", encoding="utf-8")
|
||||
print("\n".join(rows))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user