From f1dc6edb7ada643ecdd7c1f45f730aea39a91e62 Mon Sep 17 00:00:00 2001 From: slhx agent Date: Mon, 13 Jul 2026 20:06:25 +0200 Subject: [PATCH] feat(kanban): expose redacted sync diagnostics req: operations/001 req: security/002 req: sync/016 --- PLAN.md | 4 +- examples/kanban/static/command-log.js | 3 + examples/kanban/static/sync.js | 46 ++++++- examples/kanban/templates/sync_shell.heml | 1 + examples/kanban/tests/browser_e2e.rs | 145 ++++++++++++++++++++-- 5 files changed, 188 insertions(+), 11 deletions(-) diff --git a/PLAN.md b/PLAN.md index 7932b84..aa4d1ad 100644 --- a/PLAN.md +++ b/PLAN.md @@ -42,11 +42,11 @@ encryption, retention, backup, and deployment policy remain host concerns. ## Slice 4 — authoritative reconnect and convergence - [ ] **User value:** offline and concurrent work reconnects without duplicate mutation, silent loss, stale authorization, or ambiguous conflict. -- **State:** In progress — one app-owned `move_card` server command validates a durable client command id, applies the authoritative canonical column once, returns the same acknowledgement for an identical retry, rejects id reuse with a different payload, assigns one server sequence, and redelivers that canonical acknowledgement after a real EventSource disconnect/reconnect. Canonical acknowledgements and the next sequence are durably stored in a strict versioned JSON envelope using fsync plus atomic replacement; startup refuses malformed/unknown state, rebuilds the canonical board, and preserves idempotency and event replay across a real process restart. A dedicated opt-in sync route reads one pending IndexedDB command, retries transient failures with capped exponential backoff and randomized jitter, exposes online/offline state plus an accessible manual retry after exhaustion, waits for canonical acknowledgement on the reconnecting transport, and only then removes the durable command; later retry converges without a new local mutation. Canonical payload conflicts are not retried and remain durable with a visible reason. If requested history predates retained events, the stream emits a typed snapshot-required event and the client loads a versioned canonical snapshot. One deterministic rebase rule treats `reorder_card` as converged only when the canonical snapshot already places that card in `done`; it then atomically stores the snapshot/cursor and removes the satisfied command. If a later canonical command instead places the same card in `doing`, the rebase is explicitly `conflicted`, retains the local command and last committed snapshot/cursor unchanged, and exposes the divergent canonical column/reason. Two same-origin tabs coordinate an app-owned expiring IndexedDB lease so only one uploads; the standby exposes its role without issuing a request, and after the leader closes it takes over, receives one canonical acknowledgement/sequence, and removes the queue once. Each activation serializes uploads with one in flight, processes at most two acknowledged commands, exposes the retained durable count when backpressured, and resumes the next bounded run only through the visible retry action. A mixed queue commits and removes its accepted prefix exactly once, then stops on the first permanent rejection with the typed server cause visible, the rejected command plus untouched suffix durable, and blind retry disabled. The IndexedDB v1-to-v2 command migration transactionally adds the explicit target column, records a typed migration receipt, preserves causal order and interaction intent through an interrupted upload, and later drains in the original order. Every command POST now derives its principal, permission, and tenant from the current same-origin session before idempotency lookup or mutation; authenticated mode accepts only app-configured opaque session tokens and fails closed when signed out. Direct cross-tenant and stale-permission command attempts receive typed authorization denial before mutation. IndexedDB v3 additionally indexes every command and lease by the current server-derived tenant/principal partition: switched users enumerate zero foreign commands, issue no foreign replay, cannot export foreign payloads, and the owning account can export then resume its intact queue. Signed-out context lookup fails closed before opening the queue. Canonical snapshots and replayable acknowledgement history now derive the same current session, filter card state and events by tenant before serialization, use tenant-local sequence cursors, and reject signed-out reads. A divergent snapshot enables explicit “Use canonical state” and “Keep local change” actions. Canonical resolution removes only the conflicted head before resuming the ordered suffix. Keep-local resolution derives one deterministic retry identity from the original command and snapshot, keeps the original head and suffix durable through permanent rejection and bounded transport failure, re-enables the explicit decision after rejection, and removes the head only after canonical acceptance before resuming the suffix. Broader domain-specific merge choices remain. +- **State:** In progress — one app-owned `move_card` server command validates a durable client command id, applies the authoritative canonical column once, returns the same acknowledgement for an identical retry, rejects id reuse with a different payload, assigns one server sequence, and redelivers that canonical acknowledgement after a real EventSource disconnect/reconnect. Canonical acknowledgements and the next sequence are durably stored in a strict versioned JSON envelope using fsync plus atomic replacement; startup refuses malformed/unknown state, rebuilds the canonical board, and preserves idempotency and event replay across a real process restart. A dedicated opt-in sync route reads one pending IndexedDB command, retries transient failures with capped exponential backoff and randomized jitter, exposes online/offline state plus an accessible manual retry after exhaustion, waits for canonical acknowledgement on the reconnecting transport, and only then removes the durable command; later retry converges without a new local mutation. Canonical payload conflicts are not retried and remain durable with a visible reason. If requested history predates retained events, the stream emits a typed snapshot-required event and the client loads a versioned canonical snapshot. One deterministic rebase rule treats `reorder_card` as converged only when the canonical snapshot already places that card in `done`; it then atomically stores the snapshot/cursor and removes the satisfied command. If a later canonical command instead places the same card in `doing`, the rebase is explicitly `conflicted`, retains the local command and last committed snapshot/cursor unchanged, and exposes the divergent canonical column/reason. Two same-origin tabs coordinate an app-owned expiring IndexedDB lease so only one uploads; the standby exposes its role without issuing a request, and after the leader closes it takes over, receives one canonical acknowledgement/sequence, and removes the queue once. Each activation serializes uploads with one in flight, processes at most two acknowledged commands, exposes the retained durable count when backpressured, and resumes the next bounded run only through the visible retry action. A mixed queue commits and removes its accepted prefix exactly once, then stops on the first permanent rejection with the typed server cause visible, the rejected command plus untouched suffix durable, and blind retry disabled. The IndexedDB v1-to-v2 command migration transactionally adds the explicit target column, records a typed migration receipt, preserves causal order and interaction intent through an interrupted upload, and later drains in the original order. Every command POST now derives its principal, permission, and tenant from the current same-origin session before idempotency lookup or mutation; authenticated mode accepts only app-configured opaque session tokens and fails closed when signed out. Direct cross-tenant and stale-permission command attempts receive typed authorization denial before mutation. IndexedDB v3 additionally indexes every command and lease by the current server-derived tenant/principal partition: switched users enumerate zero foreign commands, issue no foreign replay, cannot export foreign payloads, and the owning account can export then resume its intact queue. Signed-out context lookup fails closed before opening the queue. Canonical snapshots and replayable acknowledgement history now derive the same current session, filter card state and events by tenant before serialization, use tenant-local sequence cursors, and reject signed-out reads. A divergent snapshot enables explicit “Use canonical state” and “Keep local change” actions. Canonical resolution removes only the conflicted head before resuming the ordered suffix. Keep-local resolution derives one deterministic retry identity from the original command and snapshot, keeps the original head and suffix durable through permanent rejection and bounded transport failure, re-enables the explicit decision after rejection, and removes the head only after canonical acceptance before resuming the suffix. The accessible sync shell exposes bounded redacted diagnostics for queue count/age bucket, cursor, acknowledgement-latency bucket, conflicts, and rejections; durable commands carry a migration-safe enqueue timestamp, while ids, payload fields, actor/session values, cookies, and exact timing stay out of diagnostics. Broader domain-specific merge choices remain. - **Build:** materialize `hemx-sync` over an integration transport with idempotent server command processing, snapshot/change cursor, durable acknowledgements, bounded ordered replay, current auth checks, rejection/conflict results, canonical replacement, reconnect jitter/backoff, multi-tab coordination, and redacted diagnostics. - **Refusals:** no default CRDT, transport in core, cached enqueue-time permission, unbounded queue, or silent last-write-wins policy. - **Requirements:** `sync/001-023`, `operations/001-005`, `security/002-005`, `performance/004-005`. -- **Proof:** `cargo test -p hemx-kanban-example --test browser_e2e idempotent_server_command_is_acknowledged_after_reconnect -- --exact` proves duplicate POST delivery yields one identical canonical acknowledgement/sequence, conflicting id reuse is rejected, EventSource reconnects after a server-closed first stream, the acknowledgement is delivered once with its sequence as event id, and a page reload shows the authoritative card in the canonical column. `cargo test -p hemx-kanban-example --test browser_e2e pending_local_command_uploads_with_bounded_retry_and_is_removed_on_ack -- --exact` proves automatic platform-store upload, one explicit transient failure, bounded exponential backoff plus jitter, reconnect acknowledgement, pending-before-ack ordering, acknowledged removal, canonical board convergence, and non-retried 409 rejection remaining durable with a visible reason. `cargo test -p hemx-kanban-example --test browser_e2e canonical_acknowledgement_survives_server_restart -- --exact` proves the versioned store is materialized before success, a real process restart reloads the same idempotent acknowledgement/sequence, EventSource replays it by id, and canonical board state is rebuilt. `cargo test -p hemx-kanban-example --test browser_e2e exhausted_offline_retries_keep_command_until_later_reconnect -- --exact` proves three bounded retries exhaust into visible offline/manual-recovery state while the command remains durable, then a later retry acknowledges/removes it and converges canonically. `cargo test -p hemx-kanban-example --test browser_e2e missing_history_rebase_and_user_conflict_resolution_preserve_suffix -- --exact` proves retained-history gap detection, typed/versioned snapshot fallback, deterministic already-canonical convergence, atomic snapshot/cursor commit, then a divergent canonical update producing explicit conflict with a two-command queue; the user’s canonical-state action removes only the conflicted head, retains and resumes the suffix, and reaches canonical convergence with an empty queue. `cargo test -p hemx-kanban-example --test browser_e2e keep_local_retry_preserves_conflicted_command_and_suffix_order -- --exact` proves the keep-local action derives one resolution identity; a permanent rejection preserves order and re-enables the choice; three later transport failures still preserve the original conflicted head plus suffix and expose manual recovery; final acceptance removes only the head and converges the untouched suffix without duplication or loss. `cargo test -p hemx-kanban-example --test browser_e2e two_tabs_coordinate_single_uploader_and_takeover_without_duplicate_application -- --exact` proves one retry-exhausted leader/one explicit standby, zero follower upload before takeover, lease takeover after the leader closes, one canonical sequence/event, one queue removal, and one board application. `cargo test -p hemx-kanban-example --test browser_e2e upload_backpressure_keeps_pending_work_visible_and_recoverable -- --exact` proves one in-flight upload, a two-acknowledgement activation limit, one retained durable command with visible recovery state, and explicit retry draining the final command without loss. `cargo test -p hemx-kanban-example --test browser_e2e mixed_queue_removes_accepted_prefix_and_retains_rejected_tail -- --exact` proves an accepted prefix is canonically applied and removed once before a permanent rejection stops processing, exposes its typed HTTP/server cause, disables blind retry, and leaves both the rejected command and untouched suffix durable. `cargo test -p hemx-kanban-example --test browser_e2e schema_upgrade_preserves_queued_order_and_local_intent -- --exact` proves a three-command v1 queue migrates atomically to the explicit-target v2 schema, remains byte-for-intent ordered after interrupted upload, then receives canonical sequences 1–3 in original order and drains without loss. `cargo test -p hemx-kanban-example --test browser_e2e account_partition_hides_replay_and_export_until_owner_returns -- --exact` proves a beta editor and alpha viewer enumerate zero commands and issue no replay for an alpha owner queue, signed-out startup cannot open a partition, no foreign id/export surface leaks, and only the returning alpha owner can export the intact command then resume it exactly once. `cargo test -p hemx-kanban-example --test browser_e2e canonical_snapshot_and_history_are_tenant_scoped -- --exact` proves alpha and beta sessions receive only their own canonical cards and acknowledgement events with independent sequence 1 cursors, while signed-out snapshot/history reads receive 401 and expose no stream. The completed slice proof must additionally cover broader domain-specific merge choices. +- **Proof:** `cargo test -p hemx-kanban-example --test browser_e2e idempotent_server_command_is_acknowledged_after_reconnect -- --exact` proves duplicate POST delivery yields one identical canonical acknowledgement/sequence, conflicting id reuse is rejected, EventSource reconnects after a server-closed first stream, the acknowledgement is delivered once with its sequence as event id, and a page reload shows the authoritative card in the canonical column. `cargo test -p hemx-kanban-example --test browser_e2e pending_local_command_uploads_with_bounded_retry_and_is_removed_on_ack -- --exact` proves automatic platform-store upload, one explicit transient failure, bounded exponential backoff plus jitter, reconnect acknowledgement, pending-before-ack ordering, acknowledged removal, canonical board convergence, and non-retried 409 rejection remaining durable with a visible reason. `cargo test -p hemx-kanban-example --test browser_e2e canonical_acknowledgement_survives_server_restart -- --exact` proves the versioned store is materialized before success, a real process restart reloads the same idempotent acknowledgement/sequence, EventSource replays it by id, and canonical board state is rebuilt. `cargo test -p hemx-kanban-example --test browser_e2e exhausted_offline_retries_keep_command_until_later_reconnect -- --exact` proves three bounded retries exhaust into visible offline/manual-recovery state while the command remains durable, then a later retry acknowledges/removes it and converges canonically. `cargo test -p hemx-kanban-example --test browser_e2e missing_history_rebase_and_user_conflict_resolution_preserve_suffix -- --exact` proves retained-history gap detection, typed/versioned snapshot fallback, deterministic already-canonical convergence, atomic snapshot/cursor commit, then a divergent canonical update producing explicit conflict with a two-command queue; the user’s canonical-state action removes only the conflicted head, retains and resumes the suffix, and reaches canonical convergence with an empty queue. `cargo test -p hemx-kanban-example --test browser_e2e keep_local_retry_preserves_conflicted_command_and_suffix_order -- --exact` proves the keep-local action derives one resolution identity; a permanent rejection preserves order and re-enables the choice; three later transport failures still preserve the original conflicted head plus suffix and expose manual recovery; final acceptance removes only the head and converges the untouched suffix without duplication or loss. `cargo test -p hemx-kanban-example --test browser_e2e redacted_sync_diagnostics_are_bounded_and_leak_no_sensitive_material -- --exact` proves a 15-second-old queue is bucketed rather than timestamped, cursor and acknowledgement latency advance after recovery, diagnostics stay under a fixed small readout, and command id/payload/actor/session/cookie material never reaches the diagnostics surface; the conflict and keep-local tests additionally assert conflict/rejection counters at their actual failure boundaries. `cargo test -p hemx-kanban-example --test browser_e2e two_tabs_coordinate_single_uploader_and_takeover_without_duplicate_application -- --exact` proves one retry-exhausted leader/one explicit standby, zero follower upload before takeover, lease takeover after the leader closes, one canonical sequence/event, one queue removal, and one board application. `cargo test -p hemx-kanban-example --test browser_e2e upload_backpressure_keeps_pending_work_visible_and_recoverable -- --exact` proves one in-flight upload, a two-acknowledgement activation limit, one retained durable command with visible recovery state, and explicit retry draining the final command without loss. `cargo test -p hemx-kanban-example --test browser_e2e mixed_queue_removes_accepted_prefix_and_retains_rejected_tail -- --exact` proves an accepted prefix is canonically applied and removed once before a permanent rejection stops processing, exposes its typed HTTP/server cause, disables blind retry, and leaves both the rejected command and untouched suffix durable. `cargo test -p hemx-kanban-example --test browser_e2e schema_upgrade_preserves_queued_order_and_local_intent -- --exact` proves a three-command v1 queue migrates atomically to the explicit-target v2 schema, remains byte-for-intent ordered after interrupted upload, then receives canonical sequences 1–3 in original order and drains without loss. `cargo test -p hemx-kanban-example --test browser_e2e account_partition_hides_replay_and_export_until_owner_returns -- --exact` proves a beta editor and alpha viewer enumerate zero commands and issue no replay for an alpha owner queue, signed-out startup cannot open a partition, no foreign id/export surface leaks, and only the returning alpha owner can export the intact command then resume it exactly once. `cargo test -p hemx-kanban-example --test browser_e2e canonical_snapshot_and_history_are_tenant_scoped -- --exact` proves alpha and beta sessions receive only their own canonical cards and acknowledgement events with independent sequence 1 cursors, while signed-out snapshot/history reads receive 401 and expose no stream. The completed slice proof must additionally cover broader domain-specific merge choices. ## Slice 5 — local-first multiplayer Kanban milestone diff --git a/examples/kanban/static/command-log.js b/examples/kanban/static/command-log.js index d81be1b..3e64608 100644 --- a/examples/kanban/static/command-log.js +++ b/examples/kanban/static/command-log.js @@ -51,6 +51,7 @@ function migrateCommandLog(request, oldVersion) { schemaVersion: COMMAND_SCHEMA, targetColumn: command.targetColumn || "done", accountPartition: command.accountPartition || "demo:demo", + queuedAt: Number.isSafeInteger(command.queuedAt) ? command.queuedAt : Date.now(), }); } meta.put({ from: oldVersion, to: DATABASE_VERSION, migrated: legacy.length }, MIGRATION_KEY); @@ -139,6 +140,7 @@ async function appendReorder(database, accountPartition, wire) { actor, session: stableSession(), causal, + queuedAt: Date.now(), kind: "reorder_card", cardId: String(wire[2] || "1"), targetColumn: "done", @@ -199,6 +201,7 @@ function validate(command) { if (typeof command.actor !== "string" || !command.actor) invalidCommand(command, "actor"); if (typeof command.session !== "string" || !command.session) invalidCommand(command, "session"); if (!Number.isSafeInteger(command.causal) || command.causal < 1) invalidCommand(command, "causal"); + if (!Number.isSafeInteger(command.queuedAt) || command.queuedAt < 0) invalidCommand(command, "queuedAt"); if (command.id !== `${command.actor}:${command.causal}`) invalidCommand(command, "id"); if (command.kind !== "reorder_card") invalidCommand(command, "kind"); if (typeof command.cardId !== "string" || !command.cardId) invalidCommand(command, "cardId"); diff --git a/examples/kanban/static/sync.js b/examples/kanban/static/sync.js index 17f269b..f85191c 100644 --- a/examples/kanban/static/sync.js +++ b/examples/kanban/static/sync.js @@ -21,6 +21,9 @@ let uploadsThisRun = 0; let uploadedTotal = 0; let inFlightUploads = 0; let maxObservedInFlight = 0; +let acknowledgementStartedAt; +let conflictCount = 0; +let rejectionCount = 0; let activeConflict; let manualRetryCommand; let stopped = false; @@ -74,6 +77,7 @@ function migrateCommandLog(request, oldVersion) { schemaVersion: COMMAND_SCHEMA, targetColumn: command.targetColumn || "done", accountPartition: command.accountPartition || "demo:demo", + queuedAt: Number.isSafeInteger(command.queuedAt) ? command.queuedAt : Date.now(), }); } meta.put({ from: oldVersion, to: DATABASE_VERSION, migrated: legacy.length }, MIGRATION_KEY); @@ -158,6 +162,34 @@ function setPhase(phase, message) { root.querySelector('[role="status"]').textContent = message; } +function ageBucket(milliseconds) { + if (milliseconds < 1000) return "lt-1s"; + if (milliseconds < 10000) return "1s-10s"; + if (milliseconds < 60000) return "10s-1m"; + return "gte-1m"; +} + +function latencyBucket(milliseconds) { + if (milliseconds < 50) return "lt-50ms"; + if (milliseconds < 250) return "50ms-250ms"; + if (milliseconds < 1000) return "250ms-1s"; + return "gte-1s"; +} + +function publishDiagnostics(commands) { + const queued = Array.isArray(commands) ? commands : []; + const oldest = queued.reduce((value, command) => { + return Number.isSafeInteger(command.queuedAt) ? Math.min(value, command.queuedAt) : value; + }, Date.now()); + root.setAttribute("data-sync-diag-queue-count", String(queued.length)); + root.setAttribute("data-sync-diag-oldest-age-bucket", queued.length === 0 ? "empty" : ageBucket(Date.now() - oldest)); + root.setAttribute("data-sync-diag-cursor", root.getAttribute("data-sync-ack-sequence") || "0"); + root.setAttribute("data-sync-diag-conflicts", String(conflictCount)); + root.setAttribute("data-sync-diag-rejections", String(rejectionCount)); + const diagnostics = root.querySelector("[data-sync-diagnostics]"); + diagnostics.textContent = `Queue ${queued.length}; oldest ${root.getAttribute("data-sync-diag-oldest-age-bucket")}; cursor ${root.getAttribute("data-sync-diag-cursor")}; acknowledgement ${root.getAttribute("data-sync-diag-ack-latency-bucket") || "none"}; conflicts ${conflictCount}; rejections ${rejectionCount}.`; +} + function validatePending(command) { if (!command || command.schemaVersion !== COMMAND_SCHEMA || command.accountPartition !== accountPartition || command.kind !== "reorder_card" || typeof command.id !== "string" || !command.id || typeof command.cardId !== "string" || !command.cardId || command.targetColumn !== "done") { throw new Error("invalid pending command"); @@ -301,6 +333,7 @@ function finishUpload() { async function continuePendingWork() { const commands = await pendingCommands(database); root.setAttribute("data-sync-pending-count", String(commands.length)); + publishDiagnostics(commands); setExportAvailable(commands.length > 0); if (commands.length === 0) return; if (uploadsThisRun >= uploadLimit) { @@ -340,6 +373,7 @@ async function synchronize(command) { } setOnline(true); root.setAttribute("data-sync-upload-sequence", String(acknowledgement.serverSequence)); + acknowledgementStartedAt = performance.now(); setPhase("awaiting-ack", `Command ${command.id} uploaded; awaiting canonical acknowledgement.`); const reconnect = command.session || command.actor || "kanban"; @@ -371,9 +405,11 @@ async function synchronize(command) { root.setAttribute("data-sync-pending-count", String(pendingAfterAck)); setExportAvailable(pendingAfterAck > 0); root.setAttribute("data-sync-ack-sequence", String(canonical.serverSequence)); - root.setAttribute("data-sync-ack-command-id", canonical.commandId); + root.setAttribute("data-sync-diag-cursor", String(canonical.serverSequence)); + root.setAttribute("data-sync-diag-ack-latency-bucket", latencyBucket(performance.now() - acknowledgementStartedAt)); root.setAttribute("data-sync-canonical-column", canonical.canonicalColumn); - setPhase("acknowledged", `Command ${command.id} acknowledged in ${canonical.canonicalColumn}.`); + publishDiagnostics(await pendingCommands(database)); + setPhase("acknowledged", `Queued change acknowledged in ${canonical.canonicalColumn}.`); root.dispatchEvent(new CustomEvent("kanban:sync-acknowledged", { detail: canonical })); synchronizing = false; clearTimeout(leaseTimer); @@ -410,7 +446,9 @@ async function synchronize(command) { setPhase("rebased", `Canonical snapshot ${snapshot.serverSequence} already satisfies ${command.id}; committed and removed the pending command.`); root.dispatchEvent(new CustomEvent("kanban:sync-rebased", { detail: { snapshot, command, decision } })); } else { + conflictCount += 1; activeConflict = { command, snapshot, decision }; + publishDiagnostics(await pendingCommands(database)); setConflictResolutionAvailable(true); setPhase("conflicted", `Canonical snapshot ${snapshot.serverSequence} conflicts with ${command.id} (${decision.reason}); the pending command remains queued.`); root.dispatchEvent(new CustomEvent("kanban:sync-conflicted", { detail: { snapshot, command, decision } })); @@ -431,6 +469,8 @@ async function synchronize(command) { root.setAttribute("data-sync-error-status", String(error.status)); const remaining = await pendingCommands(database); setManualRetryAvailable(false); + rejectionCount += 1; + publishDiagnostics(remaining); if (command.conflictResolution === "keep-local-change") { manualRetryCommand = undefined; setConflictResolutionAvailable(true); @@ -507,6 +547,8 @@ async function start() { root.setAttribute("data-sync-in-flight", "0"); root.setAttribute("data-sync-max-observed-in-flight", "0"); root.setAttribute("data-sync-pending-count", String(commands.length)); + publishDiagnostics(commands); + root.setAttribute("data-sync-diag-ack-latency-bucket", "none"); setExportAvailable(commands.length > 0); setConflictResolutionAvailable(false); setManualRetryAvailable(false); diff --git a/examples/kanban/templates/sync_shell.heml b/examples/kanban/templates/sync_shell.heml index e60b631..9457fc1 100644 --- a/examples/kanban/templates/sync_shell.heml +++ b/examples/kanban/templates/sync_shell.heml @@ -9,6 +9,7 @@

Sync status

Waiting for pending commands.

+ diff --git a/examples/kanban/tests/browser_e2e.rs b/examples/kanban/tests/browser_e2e.rs index 0e7d71f..00ccf04 100644 --- a/examples/kanban/tests/browser_e2e.rs +++ b/examples/kanban/tests/browser_e2e.rs @@ -290,7 +290,7 @@ async fn pending_local_command_uploads_with_bounded_retry_and_is_removed_on_ack( assert_eq!(proof["uploadSequence"], "1"); assert_eq!(proof["ackSequence"], "1"); assert_eq!(proof["canonicalColumn"], "done"); - assert_eq!(proof["status"], "Command sync-actor:1 acknowledged in done."); + assert_eq!(proof["status"], "Queued change acknowledged in done."); assert!(proof["error"].is_null()); let queue_count = driver @@ -580,14 +580,13 @@ async fn account_partition_hides_replay_and_export_until_owner_returns() -> WebD .await?; let authorized = driver .execute( - "const root = document.querySelector('[data-kanban-sync]'); return { sequence: root.getAttribute('data-sync-ack-sequence'), command: root.getAttribute('data-sync-ack-command-id'), column: root.getAttribute('data-sync-canonical-column'), pending: root.getAttribute('data-sync-pending-count') }", + "const root = document.querySelector('[data-kanban-sync]'); return { sequence: root.getAttribute('data-sync-ack-sequence'), column: root.getAttribute('data-sync-canonical-column'), pending: root.getAttribute('data-sync-pending-count') }", Vec::new(), ) .await? .json() .clone(); assert_eq!(authorized["sequence"], "1"); - assert_eq!(authorized["command"], "auth:1"); assert_eq!(authorized["column"], "done"); assert_eq!(authorized["pending"], "0"); assert_eq!(command_count(&driver).await?, 0); @@ -1106,7 +1105,7 @@ async fn upload_backpressure_keeps_pending_work_visible_and_recoverable() -> Web assert_eq!(recovered["total"], "3"); assert_eq!(recovered["maxInFlight"], "1"); assert_eq!(recovered["sequence"], "3"); - assert_eq!(recovered["status"], "Command pressure:3 acknowledged in done."); + assert_eq!(recovered["status"], "Queued change acknowledged in done."); assert_eq!(command_count(&driver).await?, 0); Ok(()) } @@ -1242,7 +1241,7 @@ async fn two_tabs_coordinate_single_uploader_and_takeover_without_duplicate_appl assert_eq!(takeover["attempts"], "1"); assert_eq!(takeover["pending"], "0"); assert_eq!(takeover["sequence"], "1"); - assert_eq!(takeover["status"], "Command tabs:1 acknowledged in done."); + assert_eq!(takeover["status"], "Queued change acknowledged in done."); assert_eq!(command_count(&driver).await?, 0); driver @@ -1390,7 +1389,7 @@ async fn exhausted_offline_retries_keep_command_until_later_reconnect() -> WebDr assert_eq!(converged["attempts"], "1"); assert_eq!(converged["sequence"], "1"); assert_eq!(converged["column"], "done"); - assert_eq!(converged["status"], "Command offline-actor:1 acknowledged in done."); + assert_eq!(converged["status"], "Queued change acknowledged in done."); assert!(converged["error"].is_null()); assert_eq!(command_count(&driver).await?, 0); @@ -1605,7 +1604,7 @@ async fn missing_history_rebase_and_user_conflict_resolution_preserve_suffix() - .await?; let conflicted = driver .execute( - "const root = document.querySelector('[data-kanban-sync]'); return { phase: root.getAttribute('data-sync-phase'), uploadSequence: root.getAttribute('data-sync-upload-sequence'), snapshotSequence: root.getAttribute('data-sync-snapshot-sequence'), pending: root.getAttribute('data-sync-pending-count'), rebasePending: root.getAttribute('data-sync-rebase-pending-count'), decision: root.getAttribute('data-sync-rebase-decision'), reason: root.getAttribute('data-sync-rebase-reason'), canonicalColumn: root.getAttribute('data-sync-canonical-column'), resolutionDisabled: root.querySelector('[data-sync-use-canonical]').disabled, status: root.querySelector('[role=status]').textContent, error: root.getAttribute('data-sync-error') }", + "const root = document.querySelector('[data-kanban-sync]'); return { phase: root.getAttribute('data-sync-phase'), uploadSequence: root.getAttribute('data-sync-upload-sequence'), snapshotSequence: root.getAttribute('data-sync-snapshot-sequence'), pending: root.getAttribute('data-sync-pending-count'), rebasePending: root.getAttribute('data-sync-rebase-pending-count'), decision: root.getAttribute('data-sync-rebase-decision'), reason: root.getAttribute('data-sync-rebase-reason'), canonicalColumn: root.getAttribute('data-sync-canonical-column'), resolutionDisabled: root.querySelector('[data-sync-use-canonical]').disabled, diagnosticConflicts: root.getAttribute('data-sync-diag-conflicts'), diagnosticVisible: root.querySelector('[data-sync-diagnostics]').textContent, status: root.querySelector('[role=status]').textContent, error: root.getAttribute('data-sync-error') }", Vec::new(), ) .await? @@ -1620,6 +1619,8 @@ async fn missing_history_rebase_and_user_conflict_resolution_preserve_suffix() - assert_eq!(conflicted["reason"], "canonical-state-diverged"); assert_eq!(conflicted["canonicalColumn"], "doing"); assert_eq!(conflicted["resolutionDisabled"], false); + assert_eq!(conflicted["diagnosticConflicts"], "1"); + assert!(conflicted["diagnosticVisible"].as_str().unwrap().ends_with("conflicts 1; rejections 0.")); assert_eq!( conflicted["status"], "Canonical snapshot 4 conflicts with history:3 (canonical-state-diverged); the pending command remains queued." @@ -1704,6 +1705,132 @@ async fn missing_history_rebase_and_user_conflict_resolution_preserve_suffix() - result.and(quit) } +#[tokio::test] +async fn redacted_sync_diagnostics_are_bounded_and_leak_no_sensitive_material( +) -> WebDriverResult<()> { + // test req: operations/001 req: security/002 req: sync/016 req: sync/021 + let app_port = available_port(); + let app_addr = format!("127.0.0.1:{app_port}"); + let mut app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example")); + app_command + .env("HEMX_KANBAN_ADDR", &app_addr) + .env("HEMX_KANBAN_SYNC_FAILURES", "3"); + let _app = TestProcess::start(app_command, "hemx-kanban", &app_addr, STARTUP_TIMEOUT) + .expect("start ready hemx-kanban"); + + let webdriver_port = available_port(); + let webdriver_addr = format!("127.0.0.1:{webdriver_port}"); + let mut webdriver = Command::new("geckodriver"); + webdriver.arg("--port").arg(webdriver_port.to_string()); + let _webdriver = TestProcess::start(webdriver, "geckodriver", &webdriver_addr, STARTUP_TIMEOUT) + .expect("start ready geckodriver"); + let mut caps = DesiredCapabilities::firefox(); + caps.set_headless()?; + let driver = WebDriver::new(&format!("http://{webdriver_addr}"), caps).await?; + + let result = async { + driver.goto(&format!("http://{app_addr}/")).await?; + let seeded = driver + .execute_async( + r#" + const done = arguments[arguments.length - 1]; + const open = indexedDB.open('hemx-kanban-v1', 3); + open.onupgradeneeded = () => { + const database = open.result; + const commands = database.createObjectStore('commands', { keyPath: 'id' }); + commands.createIndex('byAccountPartition', 'accountPartition'); + database.createObjectStore('meta'); + }; + open.onsuccess = () => { + const tx = open.result.transaction('commands', 'readwrite'); + tx.objectStore('commands').add({ + id: 'diag-secret-command', schemaVersion: 2, accountPartition: 'demo:demo', + actor: 'private-actor', session: 'super-secret-session-token', causal: 1, + queuedAt: Date.now() - 15000, kind: 'reorder_card', cardId: '1', targetColumn: 'done', + eventKind: 'click', key: null, privatePayload: 'customer-secret-payload', + }); + tx.oncomplete = () => done({ seeded: true }); + tx.onabort = () => done({ error: tx.error && tx.error.name }); + }; + "#, + Vec::new(), + ) + .await? + .json() + .clone(); + assert_eq!(seeded["seeded"], true); + + driver.goto(&format!("http://{app_addr}/sync-demo")).await?; + wait_until( + &driver, + "const root = document.querySelector('[data-kanban-sync]'); return root?.getAttribute('data-sync-phase') === 'offline' && root?.getAttribute('data-sync-pending-count') === '1'", + ) + .await?; + let queued = driver + .execute( + "const root = document.querySelector('[data-kanban-sync]'); const names = ['data-sync-diag-queue-count','data-sync-diag-oldest-age-bucket','data-sync-diag-cursor','data-sync-diag-ack-latency-bucket','data-sync-diag-conflicts','data-sync-diag-rejections']; const diagnostics = Object.fromEntries(names.map((name) => [name, root.getAttribute(name)])); return { diagnostics, visible: root.querySelector('[data-sync-diagnostics]').textContent }", + Vec::new(), + ) + .await? + .json() + .clone(); + assert_eq!( + queued["diagnostics"], + serde_json::json!({ + "data-sync-diag-queue-count": "1", + "data-sync-diag-oldest-age-bucket": "10s-1m", + "data-sync-diag-cursor": "0", + "data-sync-diag-ack-latency-bucket": "none", + "data-sync-diag-conflicts": "0", + "data-sync-diag-rejections": "0" + }) + ); + assert!(queued["visible"].as_str().unwrap().contains("Queue 1; oldest 10s-1m; cursor 0; acknowledgement none")); + + driver.find(By::Css("[data-sync-retry]")).await?.click().await?; + wait_until( + &driver, + "const root = document.querySelector('[data-kanban-sync]'); return root?.getAttribute('data-sync-phase') === 'acknowledged' && root?.getAttribute('data-sync-pending-count') === '0'", + ) + .await?; + let proof = driver + .execute( + "const root = document.querySelector('[data-kanban-sync]'); const names = ['data-sync-diag-queue-count','data-sync-diag-oldest-age-bucket','data-sync-diag-cursor','data-sync-diag-ack-latency-bucket','data-sync-diag-conflicts','data-sync-diag-rejections']; const diagnostics = Object.fromEntries(names.map((name) => [name, root.getAttribute(name)])); return { diagnostics, visible: root.querySelector('[data-sync-diagnostics]').textContent, rootHtml: root.outerHTML, cookies: document.cookie }", + Vec::new(), + ) + .await? + .json() + .clone(); + assert_eq!(proof["diagnostics"]["data-sync-diag-queue-count"], "0"); + assert_eq!(proof["diagnostics"]["data-sync-diag-oldest-age-bucket"], "empty"); + assert_eq!(proof["diagnostics"]["data-sync-diag-cursor"], "1"); + assert_eq!(proof["diagnostics"]["data-sync-diag-conflicts"], "0"); + assert_eq!(proof["diagnostics"]["data-sync-diag-rejections"], "0"); + assert!(matches!( + proof["diagnostics"]["data-sync-diag-ack-latency-bucket"].as_str(), + Some("lt-50ms" | "50ms-250ms" | "250ms-1s" | "gte-1s") + )); + let visible = proof["visible"].as_str().unwrap(); + assert!(visible.contains("Queue 0; oldest empty; cursor 1; acknowledgement")); + assert!(visible.len() < 120); + let exposed = format!("{}\n{}\n{}\n{}", queued["visible"], proof["visible"], proof["rootHtml"], proof["cookies"]); + for secret in [ + "diag-secret-command", + "private-actor", + "super-secret-session-token", + "customer-secret-payload", + "cardId", + "privatePayload", + ] { + assert!(!exposed.contains(secret), "diagnostics leaked {secret}: {exposed}"); + } + Ok(()) + } + .await; + let quit = driver.quit().await; + result.and(quit) +} + #[tokio::test] async fn keep_local_retry_preserves_conflicted_command_and_suffix_order() -> WebDriverResult<()> { // test req: sync/009 req: sync/010 req: sync/011 req: sync/016 @@ -1841,6 +1968,8 @@ async fn keep_local_retry_preserves_conflicted_command_and_suffix_order() -> Web resolution: root.getAttribute('data-sync-conflict-resolution'), resolutionDisabled: root.querySelector('[data-sync-keep-local]').disabled, pending: root.getAttribute('data-sync-pending-count'), + diagnosticRejections: root.getAttribute('data-sync-diag-rejections'), + diagnosticVisible: root.querySelector('[data-sync-diagnostics]').textContent, rejections: window.__keepRejections, commands: request.result.sort((left, right) => left.causal - right.causal).map(({ id, causal, cardId }) => ({ id, causal, cardId })), }); @@ -1854,6 +1983,8 @@ async fn keep_local_retry_preserves_conflicted_command_and_suffix_order() -> Web assert_eq!(rejected["resolution"], "keep-local-rejected"); assert_eq!(rejected["resolutionDisabled"], false); assert_eq!(rejected["pending"], "2"); + assert_eq!(rejected["diagnosticRejections"], "1"); + assert!(rejected["diagnosticVisible"].as_str().unwrap().ends_with("conflicts 1; rejections 1.")); assert_eq!(rejected["rejections"], 1); assert_eq!( rejected["commands"],