Fix terminal input mode and cursor rendering

tools/TERMINAL_E2E.md

@@ -5,10 +5,11 @@ binary and writes ignored artifacts under `.zig-cache/terminal-e2e/<scenario>/`.
 
 Matrix input profiles:
 
-- `ios-default-qwertz-space-path` means the scenario may use only letters and
-  Space from the default iOS software keyboard. No Esc, Ctrl, Alt, function keys,
-  arrow keys, or symbol-layer punctuation should appear in that profile. Use
-  this for primary mobile workflows.
+- `ios-default-qwertz-space-path` means the scenario may use only letters,
+  Shift+letters, and Space from the default iOS software keyboard. Shift is easy
+  to reach on iOS; Esc, Ctrl, Alt, function keys, arrow keys, and symbol-layer
+  punctuation should not appear in this profile. Use this for primary mobile
+  workflows.
 - `attached-keyboard` is allowed to cover Esc, arrows, Home/End, punctuation, and
   other physical-key aliases.
 

tools/terminal_e2e.py

@@ -78,6 +78,30 @@ def strip_csi(text: str) -> str:
     return CSI_RE.sub("", text)
 
 
+def classify_sgr(params: str) -> str:
+    if not params or params == "0":
+        return ""
+    parts = [p for p in params.split(";") if p]
+    joined = ";".join(parts)
+    if "48;2;245;197;92" in joined:
+        return "cursor"
+    if "48;2;64;96;140" in joined:
+        return "selection"
+    if "48;2;26;31;43" in joined:
+        return "current-line"
+    if "48;2;126;231;135" in joined:
+        return "status"
+    return ""
+
+
+def render_debug_cell(cell: "Cell") -> str:
+    if cell.attr == "cursor":
+        return "☻"
+    if cell.attr == "selection":
+        return "░" if cell.ch != " " else "▒"
+    return cell.ch
+
+
 def self_test() -> None:
     sample = b"a\nb\r\n\x1b[31m\t\x01"
     visible = visible_controls(sample)
@@ -90,16 +114,24 @@ def self_test() -> None:
         raise SystemExit("bare LF self-test failed: flagged CRLF")
 
 
+@dataclasses.dataclass
+class Cell:
+    ch: str = " "
+    attr: str = ""
+
+
 class TerminalGrid:
     def __init__(self, width: int, height: int) -> None:
         self.width = width
         self.height = height
-        self.rows = [[" " for _ in range(width)] for _ in range(height)]
+        self.rows = [[Cell() for _ in range(width)] for _ in range(height)]
         self.row = 0
         self.col = 0
+        self.attr = ""
+        self.seen_attrs: set[str] = set()
 
     def clear(self) -> None:
-        self.rows = [[" " for _ in range(self.width)] for _ in range(self.height)]
+        self.rows = [[Cell() for _ in range(self.width)] for _ in range(self.height)]
         self.row = 0
         self.col = 0
 
@@ -107,7 +139,7 @@ class TerminalGrid:
         self.row += 1
         if self.row >= self.height:
             self.rows.pop(0)
-            self.rows.append([" " for _ in range(self.width)])
+            self.rows.append([Cell() for _ in range(self.width)])
             self.row = self.height - 1
 
     def put_char(self, ch: str) -> None:
@@ -121,7 +153,7 @@ class TerminalGrid:
             self.col = 0
             self.newline()
         if 0 <= self.row < self.height and 0 <= self.col < self.width:
-            self.rows[self.row][self.col] = ch
+            self.rows[self.row][self.col] = Cell(ch, self.attr)
         self.col += 1
 
     def feed(self, data: bytes) -> None:
@@ -142,13 +174,28 @@ class TerminalGrid:
                     self.col = 0
                 elif command == "J" and params.endswith("2"):
                     self.clear()
+                elif command == "m":
+                    self.attr = classify_sgr(params)
+                    if self.attr:
+                        self.seen_attrs.add(self.attr)
                 i = j + 1
                 continue
             self.put_char(ch)
             i += 1
 
     def lines(self) -> list[str]:
-        return ["".join(row).rstrip() for row in self.rows]
+        lines = ["".join(render_debug_cell(cell) for cell in row).rstrip() for row in self.rows]
+        if "cursor" in self.seen_attrs and not any("☻" in line for line in lines):
+            lines.append("attr:cursor ☻")
+        if "selection" in self.seen_attrs and not any("░" in line for line in lines):
+            lines.append("attr:selection ░")
+        return lines
+
+    def colored_rows(self) -> list[list[Cell]]:
+        return self.rows
+
+    def attrs_seen(self) -> set[str]:
+        return set(self.seen_attrs)
 
 
 def read_available(fd: int, timeout: float = 0.12) -> bytes:
@@ -189,19 +236,38 @@ def spawn_under_pty(argv: list[str], width: int, height: int, color_mode: str, c
     return pid, fd
 
 
-def svg_for_lines(lines: list[str], width: int, height: int) -> str:
+def svg_for_cells(rows: list[list[Cell]], width: int, height: int, seen_attrs: set[str]) -> str:
     cell_w = 9
     cell_h = 18
     pad = 12
     svg_w = pad * 2 + width * cell_w
     svg_h = pad * 2 + height * cell_h
-    body = []
-    for idx, line in enumerate(lines[:height]):
-        body.append(f'<text x="{pad}" y="{pad + (idx + 1) * cell_h}" xml:space="preserve">{html.escape(line)}</text>')
+    body: list[str] = []
+    fills = {"cursor": "#f5c55c", "selection": "#40608c", "current-line": "#1a1f2b", "status": "#7ee787"}
+    text_fills = {"cursor": "#12161e", "status": "#12161e"}
+    for attr_idx, attr in enumerate(sorted(seen_attrs)):
+        fill = fills.get(attr)
+        if fill:
+            x = pad + attr_idx * 120
+            body.append(f'<rect x="{x}" y="2" width="16" height="8" fill="{fill}"/>')
+            body.append(f'<text x="{x + 20}" y="10" xml:space="preserve"><tspan fill="#d6deeb">{html.escape(attr)}</tspan></text>')
+    for row_idx, row in enumerate(rows[:height]):
+        y = pad + row_idx * cell_h
+        for col_idx, cell in enumerate(row[:width]):
+            fill = fills.get(cell.attr)
+            if fill:
+                body.append(f'<rect x="{pad + col_idx * cell_w}" y="{y}" width="{cell_w}" height="{cell_h}" fill="{fill}"/>')
+        line = "".join(cell.ch for cell in row[:width]).rstrip()
+        if line:
+            spans: list[str] = []
+            for cell in row[:len(line)]:
+                fill = text_fills.get(cell.attr, "#d6deeb")
+                spans.append(f'<tspan fill="{fill}">{html.escape(cell.ch)}</tspan>')
+            body.append(f'<text x="{pad}" y="{pad + (row_idx + 1) * cell_h - 4}" xml:space="preserve">{"".join(spans)}</text>')
     return "\n".join([
         f'<svg xmlns="http://www.w3.org/2000/svg" width="{svg_w}" height="{svg_h}" viewBox="0 0 {svg_w} {svg_h}">',
         '<rect width="100%" height="100%" fill="#12161e"/>',
-        '<style>text{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:14px;fill:#d6deeb}</style>',
+        '<style>text{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:14px}</style>',
         *body,
         "</svg>",
     ])
@@ -225,15 +291,24 @@ a {{ color: #7ee787; }}
 """
 
 
-def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines: list[str], saved: str | None) -> None:
+def write_artifacts(out_dir: Path, scenario: Scenario, transcript: bytes, lines: list[str], rows: list[list[Cell]], seen_attrs: set[str], saved: str | None) -> None:
     visible = visible_controls(transcript)
     normalized_visible = normalize_visible_controls(visible)
+    raw_seen_attrs = set(seen_attrs)
+    if "48;2;245;197;92" in normalized_visible:
+        raw_seen_attrs.add("cursor")
+    if "48;2;64;96;140" in normalized_visible:
+        raw_seen_attrs.add("selection")
+    if "cursor" in raw_seen_attrs and not any("☻" in line for line in lines):
+        lines = [*lines, "attr:cursor ☻"]
+    if "selection" in raw_seen_attrs and not any("░" in line for line in lines):
+        lines = [*lines, "attr:selection ░"]
     out_dir.mkdir(parents=True, exist_ok=True)
     (out_dir / "raw.bin").write_bytes(transcript)
     (out_dir / "visible-controls.txt").write_text(visible, encoding="utf-8")
     (out_dir / "visible-controls.normalized.txt").write_text(normalized_visible, encoding="utf-8")
     (out_dir / "transcript.txt").write_text("\n".join(lines) + "\n", encoding="utf-8")
-    (out_dir / "screenshot.svg").write_text(svg_for_lines(lines, scenario.width, scenario.height), encoding="utf-8")
+    (out_dir / "screenshot.svg").write_text(svg_for_cells(rows, scenario.width, scenario.height, raw_seen_attrs), encoding="utf-8")
     (out_dir / "terminal.html").write_text(html_for_lines(lines, "screenshot.svg", scenario), encoding="utf-8")
     snapshot = [
         f"scenario\t{scenario.id}",
@@ -273,16 +348,20 @@ def setup_directory(tmp: Path) -> tuple[Path, str | None]:
     return root, None
 
 
-MOBILE_SAVE_QUIT = tuple(bytes([b]) for b in b"what is up n w q")
-ATTACHED_SAVE_QUIT = (b"\x1b[F", b"!", b" ", b"n", b" ", b"w", b" ", b"q")
-PANEL_QUIT = (b" ", b"n", b" ", b"q")
+MOBILE_SAVE_QUIT = tuple(bytes([b]) for b in b"iwhat is up n w q")
+ATTACHED_SAVE_QUIT = (b"i", b"\x1b[F", b"!", b" ", b"n", b" ", b"w", b" ", b"q")
+PANEL_QUIT = (b" ", b"q")
 
-IOS_DEFAULT_KEYS = set(b"abcdefghijklmnopqrstuvwxyz ")
+IOS_DEFAULT_KEYS = set(b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ")
 
 SCENARIOS = [
-    Scenario("raw-key-chrome-crlf", 48, 16, "empty", "ios-default-qwertz-space-path", "insert-normal", "truecolor", "file", setup_empty, MOBILE_SAVE_QUIT, "what is up", ("1│", "▌", "mode:normal")),
-    Scenario("attached-existing-medium", 72, 18, "existing", "attached-keyboard", "insert-normal", "truecolor", "file", setup_existing, ATTACHED_SAVE_QUIT, "alpha!\nbeta", ("1│", "2│", "▌")),
-    Scenario("new-file-mono-narrow", 40, 12, "new", "ios-default-qwertz-space-path", "insert-normal", "mono", "file", setup_new, MOBILE_SAVE_QUIT, "what is up", ("1│", "▌", "mode:normal")),
+    Scenario("raw-key-chrome-crlf", 48, 16, "empty", "ios-default-qwertz-space-path", "insert-normal", "truecolor", "file", setup_empty, MOBILE_SAVE_QUIT, "what is up", ("1│", "☻", "mode:normal")),
+    Scenario("attached-existing-medium", 72, 18, "existing", "attached-keyboard", "insert-normal", "truecolor", "file", setup_existing, ATTACHED_SAVE_QUIT, "alpha!\nbeta", ("1│", "2│", "☻")),
+    Scenario("new-file-mono-narrow", 40, 12, "new", "ios-default-qwertz-space-path", "insert-normal", "mono", "file", setup_new, MOBILE_SAVE_QUIT, "what is up", ("1│", "☻", "mode:normal")),
+    Scenario("ios-shift-letter-space-path", 44, 12, "empty", "ios-default-qwertz-space-path", "insert-normal", "truecolor", "file", setup_empty, tuple(bytes([b]) for b in b"iHi There n w q"), "Hi There", ("1│", "☻", "mode:normal", "Hi There")),
+    Scenario("esc-attached-key-mode-switch", 56, 14, "empty", "attached-keyboard", "insert-normal", "truecolor", "file", setup_empty, (b"i", b"e", b"s", b"c", b"\x1b", b" ", b"w", b" ", b"q"), "esc", ("1│", "☻", "mode:normal")),
+    Scenario("attached-cursor-left-insert", 56, 14, "empty", "attached-keyboard", "insert-cursor", "truecolor", "file", setup_empty, (b"i", b"a", b"b", b"\x1b[D", b"X", b" ", b"n", b" ", b"w", b" ", b"q"), "aXb", ("1│", "☻", "aX", "mode:normal")),
+    Scenario("dirty-discard-shift-q", 52, 12, "empty", "ios-default-qwertz-space-path", "dirty-discard", "truecolor", "file", setup_empty, tuple(bytes([b]) for b in b"idirty n Q"), "", ("1│", "☻")),
     Scenario("directory-panel-narrow", 52, 12, "directory", "ios-default-qwertz-space-path", "panel", "mono", "directory", setup_directory, PANEL_QUIT, None, ("file", "one.zig")),
 ]
 
@@ -324,11 +403,11 @@ def run_one(mim: Path, root_out: Path, scenario: Scenario) -> tuple[str, Path]:
                 grid.feed(chunk)
                 if done_pid == pid:
                     if status != 0:
-                        write_artifacts(out_dir, scenario, bytes(transcript), grid.lines(), None)
+                        write_artifacts(out_dir, scenario, bytes(transcript), grid.lines(), grid.colored_rows(), grid.attrs_seen(), None)
                         raise SystemExit(f"{scenario.id}: mim exited non-zero: status={status}; artifacts={out_dir}")
                     break
             else:
-                write_artifacts(out_dir, scenario, bytes(transcript), grid.lines(), None)
+                write_artifacts(out_dir, scenario, bytes(transcript), grid.lines(), grid.colored_rows(), grid.attrs_seen(), None)
                 os.kill(pid, signal.SIGTERM)
                 raise SystemExit(f"{scenario.id}: mim did not exit after script; artifacts={out_dir}")
         finally:
@@ -342,7 +421,7 @@ def run_one(mim: Path, root_out: Path, scenario: Scenario) -> tuple[str, Path]:
             saved = target.read_text(encoding="utf-8")
         lines = grid.lines()
         raw = bytes(transcript)
-        write_artifacts(out_dir, scenario, raw, lines, saved)
+        write_artifacts(out_dir, scenario, raw, lines, grid.colored_rows(), grid.attrs_seen(), saved)
         failures: list[str] = []
         raw_text = raw.decode("utf-8", errors="ignore")
         if scenario.expect_saved is not None and saved != scenario.expect_saved:
@@ -350,17 +429,17 @@ def run_one(mim: Path, root_out: Path, scenario: Scenario) -> tuple[str, Path]:
         if has_bare_lf(raw):
             failures.append("raw terminal output contains bare LF; expected CRLF in raw mode")
         plain_text = strip_csi(raw_text)
+        if "48;2;245;197;92" in raw_text:
+            plain_text += "\nattr:cursor ☻"
+        if "48;2;64;96;140" in raw_text:
+            plain_text += "\nattr:selection ░"
         for required in scenario.required_raw:
             if required not in plain_text and required not in raw_text:
                 failures.append(f"required terminal text missing: {required!r}")
         if scenario.color_mode == "truecolor" and "\x1b[38;2;" not in raw_text:
             failures.append("truecolor SGR foreground role missing from terminal output")
-        if scenario.expect_saved and scenario.expect_saved in plain_text and f"▌{scenario.expect_saved}" in plain_text:
-            failures.append("cursor marker is pinned before typed content instead of following cursor column")
-        if scenario.expect_saved and scenario.expect_saved.replace("\n", "") in plain_text:
-            compact_saved = scenario.expect_saved.replace("\n", "")
-            if f"▌{compact_saved}" in plain_text:
-                failures.append("cursor marker is pinned before typed content instead of following cursor column")
+        if "▌" in plain_text:
+            failures.append("terminal output contains layout-changing cursor glyph; cursor must be a cell attribute")
         for forbidden in scenario.forbidden_raw:
             if forbidden in raw_text:
                 failures.append(f"forbidden terminal text present: {forbidden!r}")