use hemx_test::TestProcess;
use std::fs;
use std::net::TcpListener;
use std::process::Command;
use std::time::Duration;
use thirtyfour::prelude::*;

const STARTUP_TIMEOUT: Duration = Duration::from_secs(12);

#[tokio::test]
async fn server_first_route_does_not_load_optional_client_assets() -> WebDriverResult<()> {
    // test req: performance/006
    let app_port = available_port();
    let app_addr = format!("127.0.0.1:{app_port}");
    let mut app = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
    app.env("HEMX_KANBAN_ADDR", &app_addr);
    let _app = TestProcess::start(app, "hemx-kanban", &app_addr, STARTUP_TIMEOUT)
        .expect("start ready hemx-kanban");

    let webdriver_port = available_port();
    let webdriver_addr = format!("127.0.0.1:{webdriver_port}");
    let mut webdriver = Command::new("geckodriver");
    webdriver.arg("--port").arg(webdriver_port.to_string());
    let _webdriver = TestProcess::start(webdriver, "geckodriver", &webdriver_addr, STARTUP_TIMEOUT)
        .expect("start ready geckodriver");

    let mut caps = DesiredCapabilities::firefox();
    caps.set_headless()?;
    let driver = WebDriver::new(&format!("http://{webdriver_addr}"), caps).await?;
    let result = async {
        driver.goto(&format!("http://{app_addr}/")).await?;
        driver.find(By::Css("section[data-hemx-root='kanban']")).await?;
        let loaded = driver
            .execute_async(
                r#"
                const done = arguments[arguments.length - 1];
                navigator.serviceWorker.getRegistrations().then(async (registrations) => {
                  const databases = indexedDB.databases ? await indexedDB.databases() : [];
                  done({
                    resources: performance.getEntriesByType('resource').map((entry) => new URL(entry.name).pathname),
                    scripts: [...document.scripts].map((script) => ({ src: new URL(script.src).pathname, type: script.type })),
                    clientRoots: document.querySelectorAll('[data-hemx-client-module]').length,
                    serviceWorkers: registrations.length,
                    databases: databases.map((database) => database.name),
                  });
                }).catch((error) => done({ error: String(error) }));
                "#,
                Vec::new(),
            )
            .await?
            .json()
            .clone();

        assert!(loaded["error"].is_null(), "browser inspection failed: {loaded}");
        assert_eq!(loaded["clientRoots"], 0);
        assert_eq!(loaded["serviceWorkers"], 0);
        assert_eq!(loaded["databases"].as_array().map(Vec::len), Some(0));
        let scripts = loaded["scripts"].as_array().expect("document scripts");
        assert_eq!(scripts.len(), 1);
        let runtime_path = scripts[0]["src"].as_str().expect("runtime script path");
        assert!(
            runtime_path.starts_with("/hemx.") && runtime_path.ends_with(".js"),
            "unexpected server runtime asset: {loaded}"
        );
        assert_eq!(scripts[0]["type"], "");
        let resources = loaded["resources"]
            .as_array()
            .expect("resource timing entries")
            .iter()
            .filter_map(|value| value.as_str())
            .collect::<Vec<_>>();
        assert!(resources.contains(&runtime_path), "runtime was not loaded: {loaded}");
        for optional in [
            "/hemx.client.js",
            "/kanban_client.js",
            "/kanban_client_bg.wasm",
            "/app.js",
            "/offline.js",
        ] {
            assert!(
                !resources.contains(&optional),
                "server-first route loaded optional client asset {optional}: {loaded}"
            );
        }
        Ok(())
    }
    .await;
    let quit = driver.quit().await;
    result.and(quit)
}

#[tokio::test]
async fn idempotent_server_command_is_acknowledged_after_reconnect() -> WebDriverResult<()> {
    // test req: sync/001 req: sync/005 req: sync/006 req: sync/007
    // test req: sync/008 req: sync/012 req: sync/013
    let app_port = available_port();
    let app_addr = format!("127.0.0.1:{app_port}");
    let mut app = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
    app.env("HEMX_KANBAN_ADDR", &app_addr);
    let _app = TestProcess::start(app, "hemx-kanban", &app_addr, STARTUP_TIMEOUT)
        .expect("start ready hemx-kanban");

    let webdriver_port = available_port();
    let webdriver_addr = format!("127.0.0.1:{webdriver_port}");
    let mut webdriver = Command::new("geckodriver");
    webdriver.arg("--port").arg(webdriver_port.to_string());
    let _webdriver = TestProcess::start(webdriver, "geckodriver", &webdriver_addr, STARTUP_TIMEOUT)
        .expect("start ready geckodriver");

    let mut caps = DesiredCapabilities::firefox();
    caps.set_headless()?;
    let driver = WebDriver::new(&format!("http://{webdriver_addr}"), caps).await?;
    let result = async {
        driver.goto(&format!("http://{app_addr}/")).await?;
        driver.find(By::Css("section[data-hemx-root='kanban']")).await?;
        driver
            .execute(
                r#"
                window.__syncProof = { ready: false, opens: 0, events: [], error: null };
                (async () => {
                  const endpoint = '/sync/commands?command_id=actor-1%3A1&card_id=1';
                  const firstResponse = await fetch(endpoint, { method: 'POST' });
                  const first = await firstResponse.json();
                  const duplicateResponse = await fetch(endpoint, { method: 'POST' });
                  const duplicate = await duplicateResponse.json();
                  const conflictResponse = await fetch('/sync/commands?command_id=actor-1%3A1&card_id=2', { method: 'POST' });
                  const conflict = await conflictResponse.json();
                  window.__syncProof.command = {
                    firstStatus: firstResponse.status,
                    duplicateStatus: duplicateResponse.status,
                    first,
                    duplicate,
                    conflictStatus: conflictResponse.status,
                    conflict,
                  };
                  const source = new EventSource('/sync/acknowledgements?after=0&reconnect=browser-proof');
                  source.onopen = () => { window.__syncProof.opens += 1; };
                  source.addEventListener('acknowledgement', (event) => {
                    window.__syncProof.events.push({ id: event.lastEventId, acknowledgement: JSON.parse(event.data) });
                    window.__syncProof.ready = true;
                    source.close();
                  });
                  source.onerror = () => {
                    if (source.readyState === EventSource.CLOSED && !window.__syncProof.ready) {
                      window.__syncProof.error = 'acknowledgement stream closed';
                    }
                  };
                })().catch((error) => { window.__syncProof.error = String(error); });
                return true;
                "#,
                Vec::new(),
            )
            .await?;
        wait_until(
            &driver,
            "return window.__syncProof.ready === true || window.__syncProof.error !== null",
        )
        .await?;
        let proof = driver
            .execute("return window.__syncProof", Vec::new())
            .await?
            .json()
            .clone();
        assert!(proof["error"].is_null(), "sync failed: {proof}");
        assert!(
            proof["opens"].as_u64().is_some_and(|opens| opens >= 2),
            "transport did not reconnect: {proof}"
        );
        assert_eq!(proof["command"]["firstStatus"], 200);
        assert_eq!(proof["command"]["duplicateStatus"], 200);
        assert_eq!(proof["command"]["first"], proof["command"]["duplicate"]);
        assert_eq!(proof["command"]["first"]["commandId"], "actor-1:1");
        assert_eq!(proof["command"]["first"]["serverSequence"], 1);
        assert_eq!(proof["command"]["first"]["cardId"], 1);
        assert_eq!(proof["command"]["first"]["canonicalColumn"], "done");
        assert_eq!(proof["command"]["first"]["status"], "accepted");
        assert_eq!(proof["command"]["conflictStatus"], 409);
        assert_eq!(
            proof["command"]["conflict"]["error"],
            "command_id was already used for a different payload"
        );
        assert_eq!(proof["events"].as_array().map(Vec::len), Some(1));
        assert_eq!(proof["events"][0]["id"], "1");
        assert_eq!(
            proof["events"][0]["acknowledgement"],
            proof["command"]["first"]
        );

        driver.refresh().await?;
        let canonical = driver
            .execute(
                "return [...document.querySelectorAll('section.column')].map((column) => ({ title: column.querySelector('h2').textContent, cards: [...column.querySelectorAll('[data-key]')].map((card) => card.dataset.key) }))",
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(canonical[2]["title"], "Done");
        assert_eq!(canonical[2]["cards"], serde_json::json!(["1", "3"]));
        Ok(())
    }
    .await;
    let quit = driver.quit().await;
    result.and(quit)
}

#[tokio::test]
async fn pending_local_command_uploads_with_bounded_retry_and_is_removed_on_ack(
) -> WebDriverResult<()> {
    // test req: sync/004 req: sync/009 req: sync/010 req: sync/016 req: sync/017
    let app_port = available_port();
    let app_addr = format!("127.0.0.1:{app_port}");
    let mut app = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
    app.env("HEMX_KANBAN_ADDR", &app_addr)
        .env("HEMX_KANBAN_FAIL_FIRST_SYNC", "1");
    let _app = TestProcess::start(app, "hemx-kanban", &app_addr, STARTUP_TIMEOUT)
        .expect("start ready hemx-kanban");

    let webdriver_port = available_port();
    let webdriver_addr = format!("127.0.0.1:{webdriver_port}");
    let mut webdriver = Command::new("geckodriver");
    webdriver.arg("--port").arg(webdriver_port.to_string());
    let _webdriver = TestProcess::start(webdriver, "geckodriver", &webdriver_addr, STARTUP_TIMEOUT)
        .expect("start ready geckodriver");

    let mut caps = DesiredCapabilities::firefox();
    caps.set_headless()?;
    let driver = WebDriver::new(&format!("http://{webdriver_addr}"), caps).await?;
    let result = async {
        driver.goto(&format!("http://{app_addr}/")).await?;
        let seeded = driver
            .execute_async(
                r#"
                const done = arguments[arguments.length - 1];
                const open = indexedDB.open('hemx-kanban-v1', 1);
                open.onupgradeneeded = () => {
                  const database = open.result;
                  if (!database.objectStoreNames.contains('commands')) database.createObjectStore('commands', { keyPath: 'id' });
                  if (!database.objectStoreNames.contains('meta')) database.createObjectStore('meta');
                };
                open.onerror = () => done({ error: open.error && open.error.name });
                open.onsuccess = () => {
                  const tx = open.result.transaction('commands', 'readwrite');
                  tx.objectStore('commands').add({
                    id: 'sync-actor:1', schemaVersion: 1, actor: 'sync-actor', session: 'sync-session',
                    causal: 1, kind: 'reorder_card', cardId: '1', eventKind: 'click', key: null,
                  });
                  tx.oncomplete = () => done({ seeded: true });
                  tx.onabort = () => done({ error: tx.error && tx.error.name });
                };
                "#,
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(seeded["seeded"], true, "failed to seed durable command: {seeded}");

        driver.goto(&format!("http://{app_addr}/sync-demo")).await?;
        wait_until(
            &driver,
            "return document.querySelector('[data-kanban-sync]')?.getAttribute('data-sync-phase') === 'acknowledged' || document.querySelector('[data-kanban-sync]')?.getAttribute('data-sync-phase') === 'failed'",
        )
        .await?;
        let proof = driver
            .execute(
                "const root = document.querySelector('[data-kanban-sync]'); return { phase: root.getAttribute('data-sync-phase'), pending: root.getAttribute('data-sync-pending-count'), pendingBeforeAck: root.getAttribute('data-sync-pending-before-ack'), attempts: root.getAttribute('data-sync-attempts'), maxAttempts: root.getAttribute('data-sync-max-attempts'), backoffBase: root.getAttribute('data-sync-last-backoff-base-ms'), backoff: root.getAttribute('data-sync-last-backoff-ms'), opens: root.getAttribute('data-sync-transport-opens'), uploadSequence: root.getAttribute('data-sync-upload-sequence'), ackSequence: root.getAttribute('data-sync-ack-sequence'), canonicalColumn: root.getAttribute('data-sync-canonical-column'), status: root.querySelector('[role=status]').textContent, error: root.getAttribute('data-sync-error') }",
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(proof["phase"], "acknowledged", "sync failed: {proof}");
        assert_eq!(proof["pending"], "0");
        assert_eq!(proof["pendingBeforeAck"], "1");
        assert_eq!(proof["attempts"], "2");
        assert_eq!(proof["maxAttempts"], "3");
        assert_eq!(proof["backoffBase"], "25");
        assert!(
            proof["backoff"]
                .as_str()
                .and_then(|value| value.parse::<u64>().ok())
                .is_some_and(|delay| (25..50).contains(&delay)),
            "retry jitter left its bounded interval: {proof}"
        );
        assert!(
            proof["opens"].as_str().and_then(|value| value.parse::<u64>().ok()).is_some_and(|opens| opens >= 2),
            "transport did not reconnect: {proof}"
        );
        assert_eq!(proof["uploadSequence"], "1");
        assert_eq!(proof["ackSequence"], "1");
        assert_eq!(proof["canonicalColumn"], "done");
        assert_eq!(proof["status"], "Command sync-actor:1 acknowledged in done.");
        assert!(proof["error"].is_null());

        let queue_count = driver
            .execute_async(
                r#"
                const done = arguments[arguments.length - 1];
                const open = indexedDB.open('hemx-kanban-v1', 1);
                open.onsuccess = () => {
                  const request = open.result.transaction('commands', 'readonly').objectStore('commands').count();
                  request.onsuccess = () => done(request.result);
                  request.onerror = () => done({ error: request.error && request.error.name });
                };
                "#,
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(queue_count, 0);

        let rejected_seed = driver
            .execute_async(
                r#"
                const done = arguments[arguments.length - 1];
                const open = indexedDB.open('hemx-kanban-v1', 1);
                open.onsuccess = () => {
                  const tx = open.result.transaction('commands', 'readwrite');
                  tx.objectStore('commands').add({
                    id: 'sync-actor:1', schemaVersion: 1, actor: 'sync-actor', session: 'sync-session',
                    causal: 2, kind: 'reorder_card', cardId: '2', eventKind: 'click', key: null,
                  });
                  tx.oncomplete = () => done({ seeded: true });
                  tx.onabort = () => done({ error: tx.error && tx.error.name });
                };
                "#,
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(rejected_seed["seeded"], true, "failed to seed rejected command");
        driver.goto(&format!("http://{app_addr}/sync-demo")).await?;
        wait_until(
            &driver,
            "return document.querySelector('[data-kanban-sync]')?.getAttribute('data-sync-phase') === 'failed'",
        )
        .await?;
        let rejected = driver
            .execute(
                "const root = document.querySelector('[data-kanban-sync]'); return { pending: root.getAttribute('data-sync-pending-count'), attempts: root.getAttribute('data-sync-attempts'), error: root.getAttribute('data-sync-error'), status: root.querySelector('[role=status]').textContent }",
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(rejected["pending"], "1");
        assert_eq!(rejected["attempts"], "1");
        assert_eq!(rejected["error"], "sync upload failed with 409");
        assert_eq!(rejected["status"], "Sync failed; the durable command remains queued.");

        driver.goto(&format!("http://{app_addr}/")).await?;
        let canonical = driver
            .execute(
                "return [...document.querySelectorAll('section.column')].map((column) => ({ title: column.querySelector('h2').textContent, cards: [...column.querySelectorAll('[data-key]')].map((card) => card.dataset.key) }))",
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(canonical[2]["title"], "Done");
        assert_eq!(canonical[2]["cards"], serde_json::json!(["1", "3"]));
        Ok(())
    }
    .await;
    let quit = driver.quit().await;
    result.and(quit)
}

#[tokio::test]
async fn exhausted_offline_retries_keep_command_until_later_reconnect() -> WebDriverResult<()> {
    // test req: sync/004 req: sync/010 req: sync/011 req: sync/014 req: sync/016
    let app_port = available_port();
    let app_addr = format!("127.0.0.1:{app_port}");
    let mut app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
    app_command
        .env("HEMX_KANBAN_ADDR", &app_addr)
        .env("HEMX_KANBAN_SYNC_FAILURES", "3");
    let _app = TestProcess::start(app_command, "hemx-kanban", &app_addr, STARTUP_TIMEOUT)
        .expect("start ready hemx-kanban");

    let webdriver_port = available_port();
    let webdriver_addr = format!("127.0.0.1:{webdriver_port}");
    let mut webdriver = Command::new("geckodriver");
    webdriver.arg("--port").arg(webdriver_port.to_string());
    let _webdriver = TestProcess::start(webdriver, "geckodriver", &webdriver_addr, STARTUP_TIMEOUT)
        .expect("start ready geckodriver");
    let mut caps = DesiredCapabilities::firefox();
    caps.set_headless()?;
    let driver = WebDriver::new(&format!("http://{webdriver_addr}"), caps).await?;

    let result = async {
        driver.goto(&format!("http://{app_addr}/")).await?;
        let seeded = driver
            .execute_async(
                r#"
                const done = arguments[arguments.length - 1];
                const open = indexedDB.open('hemx-kanban-v1', 1);
                open.onupgradeneeded = () => {
                  const database = open.result;
                  if (!database.objectStoreNames.contains('commands')) database.createObjectStore('commands', { keyPath: 'id' });
                  if (!database.objectStoreNames.contains('meta')) database.createObjectStore('meta');
                };
                open.onsuccess = () => {
                  const tx = open.result.transaction('commands', 'readwrite');
                  tx.objectStore('commands').add({
                    id: 'offline-actor:1', schemaVersion: 1, actor: 'offline-actor', session: 'offline-session',
                    causal: 1, kind: 'reorder_card', cardId: '2', eventKind: 'click', key: null,
                  });
                  tx.oncomplete = () => done({ seeded: true });
                  tx.onabort = () => done({ error: tx.error && tx.error.name });
                };
                "#,
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(seeded["seeded"], true, "failed to seed offline command: {seeded}");

        driver.goto(&format!("http://{app_addr}/sync-demo")).await?;
        wait_until(
            &driver,
            "return document.querySelector('[data-kanban-sync]')?.getAttribute('data-sync-phase') === 'offline'",
        )
        .await?;
        let exhausted = driver
            .execute(
                "const root = document.querySelector('[data-kanban-sync]'); return { phase: root.getAttribute('data-sync-phase'), connection: root.getAttribute('data-sync-connection'), pending: root.getAttribute('data-sync-pending-count'), attempts: root.getAttribute('data-sync-attempts'), maxAttempts: root.getAttribute('data-sync-max-attempts'), manual: root.getAttribute('data-sync-manual-retry'), error: root.getAttribute('data-sync-error'), status: root.querySelector('[role=status]').textContent, retry: root.querySelector('[data-sync-retry]').textContent }",
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(exhausted["phase"], "offline");
        assert_eq!(exhausted["connection"], "offline");
        assert_eq!(exhausted["pending"], "1");
        assert_eq!(exhausted["attempts"], "3");
        assert_eq!(exhausted["maxAttempts"], "3");
        assert_eq!(exhausted["manual"], "available");
        assert_eq!(exhausted["error"], "sync upload failed with 503");
        assert_eq!(
            exhausted["status"],
            "Sync is offline after bounded retries; the durable command remains queued. Retry now when ready."
        );
        assert_eq!(exhausted["retry"], "Retry sync now");
        let queued = command_count(&driver).await?;
        assert_eq!(queued, 1);

        driver
            .find(By::Css("[data-sync-retry]"))
            .await?
            .click()
            .await?;
        wait_until(
            &driver,
            "return document.querySelector('[data-kanban-sync]')?.getAttribute('data-sync-phase') === 'acknowledged'",
        )
        .await?;
        let converged = driver
            .execute(
                "const root = document.querySelector('[data-kanban-sync]'); return { connection: root.getAttribute('data-sync-connection'), pending: root.getAttribute('data-sync-pending-count'), attempts: root.getAttribute('data-sync-attempts'), sequence: root.getAttribute('data-sync-ack-sequence'), column: root.getAttribute('data-sync-canonical-column'), status: root.querySelector('[role=status]').textContent, error: root.getAttribute('data-sync-error') }",
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(converged["connection"], "online");
        assert_eq!(converged["pending"], "0");
        assert_eq!(converged["attempts"], "1");
        assert_eq!(converged["sequence"], "1");
        assert_eq!(converged["column"], "done");
        assert_eq!(converged["status"], "Command offline-actor:1 acknowledged in done.");
        assert!(converged["error"].is_null());
        assert_eq!(command_count(&driver).await?, 0);

        driver.goto(&format!("http://{app_addr}/")).await?;
        let canonical = driver
            .execute(
                "return [...document.querySelectorAll('section.column')].map((column) => ({ title: column.querySelector('h2').textContent, cards: [...column.querySelectorAll('[data-key]')].map((card) => card.dataset.key) }))",
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(canonical[2]["title"], "Done");
        assert_eq!(canonical[2]["cards"], serde_json::json!(["2", "3"]));
        Ok(())
    }
    .await;
    let quit = driver.quit().await;
    result.and(quit)
}

#[tokio::test]
async fn missing_history_loads_snapshot_and_preserves_pending_intent() -> WebDriverResult<()> {
    // test req: sync/007 req: sync/020
    let app_port = available_port();
    let app_addr = format!("127.0.0.1:{app_port}");
    let mut app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
    app_command
        .env("HEMX_KANBAN_ADDR", &app_addr)
        .env("HEMX_KANBAN_RETAINED_AFTER", "1");
    let _app = TestProcess::start(app_command, "hemx-kanban", &app_addr, STARTUP_TIMEOUT)
        .expect("start ready hemx-kanban");

    let webdriver_port = available_port();
    let webdriver_addr = format!("127.0.0.1:{webdriver_port}");
    let mut webdriver = Command::new("geckodriver");
    webdriver.arg("--port").arg(webdriver_port.to_string());
    let _webdriver = TestProcess::start(webdriver, "geckodriver", &webdriver_addr, STARTUP_TIMEOUT)
        .expect("start ready geckodriver");
    let mut caps = DesiredCapabilities::firefox();
    caps.set_headless()?;
    let driver = WebDriver::new(&format!("http://{webdriver_addr}"), caps).await?;

    let result = async {
        driver.goto(&format!("http://{app_addr}/")).await?;
        let seeded_server = driver
            .execute_async(
                r#"
                const done = arguments[arguments.length - 1];
                fetch('/sync/commands?command_id=history%3A1&card_id=1', { method: 'POST' })
                  .then(async (response) => done({ status: response.status, body: await response.json() }))
                  .catch((error) => done({ error: String(error) }));
                "#,
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(seeded_server["status"], 200);
        assert_eq!(seeded_server["body"]["serverSequence"], 1);

        let seeded_local = driver
            .execute_async(
                r#"
                const done = arguments[arguments.length - 1];
                const open = indexedDB.open('hemx-kanban-v1', 1);
                open.onupgradeneeded = () => {
                  const database = open.result;
                  if (!database.objectStoreNames.contains('commands')) database.createObjectStore('commands', { keyPath: 'id' });
                  if (!database.objectStoreNames.contains('meta')) database.createObjectStore('meta');
                };
                open.onsuccess = () => {
                  const tx = open.result.transaction('commands', 'readwrite');
                  tx.objectStore('commands').add({
                    id: 'history:2', schemaVersion: 1, actor: 'history', session: 'history-session',
                    causal: 2, kind: 'reorder_card', cardId: '2', eventKind: 'click', key: null,
                  });
                  tx.oncomplete = () => done({ seeded: true });
                  tx.onabort = () => done({ error: tx.error && tx.error.name });
                };
                "#,
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(seeded_local["seeded"], true);

        driver.goto(&format!("http://{app_addr}/sync-demo")).await?;
        wait_until(
            &driver,
            "return document.querySelector('[data-kanban-sync]')?.getAttribute('data-sync-phase') === 'rebase-required'",
        )
        .await?;
        let fallback = driver
            .execute(
                "const root = document.querySelector('[data-kanban-sync]'); return { phase: root.getAttribute('data-sync-phase'), uploadSequence: root.getAttribute('data-sync-upload-sequence'), snapshotSequence: root.getAttribute('data-sync-snapshot-sequence'), snapshotSchema: root.getAttribute('data-sync-snapshot-schema'), snapshotCards: root.getAttribute('data-sync-snapshot-card-count'), pending: root.getAttribute('data-sync-pending-count'), rebasePending: root.getAttribute('data-sync-rebase-pending-count'), canonicalColumn: root.getAttribute('data-sync-canonical-column'), status: root.querySelector('[role=status]').textContent, error: root.getAttribute('data-sync-error') }",
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(fallback["phase"], "rebase-required");
        assert_eq!(fallback["uploadSequence"], "2");
        assert_eq!(fallback["snapshotSequence"], "2");
        assert_eq!(fallback["snapshotSchema"], "1");
        assert_eq!(fallback["snapshotCards"], "3");
        assert_eq!(fallback["pending"], "1");
        assert_eq!(fallback["rebasePending"], "1");
        assert_eq!(fallback["canonicalColumn"], "done");
        assert_eq!(
            fallback["status"],
            "History is unavailable; loaded snapshot 2 and preserved 1 pending command for rebase."
        );
        assert!(fallback["error"].is_null());
        assert_eq!(command_count(&driver).await?, 1);

        driver.goto(&format!("http://{app_addr}/")).await?;
        let canonical = driver
            .execute(
                "return [...document.querySelectorAll('section.column')].map((column) => ({ title: column.querySelector('h2').textContent, cards: [...column.querySelectorAll('[data-key]')].map((card) => card.dataset.key) }))",
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(canonical[2]["title"], "Done");
        assert_eq!(canonical[2]["cards"], serde_json::json!(["1", "2", "3"]));
        Ok(())
    }
    .await;
    let quit = driver.quit().await;
    result.and(quit)
}

#[tokio::test]
async fn canonical_acknowledgement_survives_server_restart() -> WebDriverResult<()> {
    // test req: sync/001 req: sync/005 req: sync/007 req: sync/008 req: sync/013
    let app_port = available_port();
    let app_addr = format!("127.0.0.1:{app_port}");
    let store = std::env::temp_dir().join(format!(
        "hemx-kanban-sync-{}-{app_port}.json",
        std::process::id()
    ));
    let _ = fs::remove_file(&store);

    let mut first_app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
    first_app_command
        .env("HEMX_KANBAN_ADDR", &app_addr)
        .env("HEMX_KANBAN_SYNC_STORE", &store);
    let first_app = TestProcess::start(
        first_app_command,
        "hemx-kanban-first",
        &app_addr,
        STARTUP_TIMEOUT,
    )
    .expect("start first hemx-kanban");

    let webdriver_port = available_port();
    let webdriver_addr = format!("127.0.0.1:{webdriver_port}");
    let mut webdriver = Command::new("geckodriver");
    webdriver.arg("--port").arg(webdriver_port.to_string());
    let _webdriver = TestProcess::start(webdriver, "geckodriver", &webdriver_addr, STARTUP_TIMEOUT)
        .expect("start ready geckodriver");
    let mut caps = DesiredCapabilities::firefox();
    caps.set_headless()?;
    let driver = WebDriver::new(&format!("http://{webdriver_addr}"), caps).await?;

    let result = async {
        driver.goto(&format!("http://{app_addr}/")).await?;
        let accepted = driver
            .execute_async(
                r#"
                const done = arguments[arguments.length - 1];
                fetch('/sync/commands?command_id=restart-proof%3A1&card_id=1', { method: 'POST' })
                  .then(async (response) => done({ status: response.status, body: await response.json() }))
                  .catch((error) => done({ error: String(error) }));
                "#,
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(accepted["status"], 200);
        assert_eq!(accepted["body"]["serverSequence"], 1);
        assert_eq!(accepted["body"]["canonicalColumn"], "done");
        assert!(store.is_file(), "server did not materialize sync store");
        let persisted = fs::read_to_string(&store).expect("read sync store");
        assert!(persisted.contains("restart-proof:1"));
        assert!(persisted.contains("\"schemaVersion\": 1"));

        drop(first_app);
        let mut second_app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
        second_app_command
            .env("HEMX_KANBAN_ADDR", &app_addr)
            .env("HEMX_KANBAN_SYNC_STORE", &store);
        let second_app = TestProcess::start(
            second_app_command,
            "hemx-kanban-second",
            &app_addr,
            STARTUP_TIMEOUT,
        )
        .expect("restart hemx-kanban from durable sync store");

        let after_restart = driver
            .execute_async(
                r#"
                const done = arguments[arguments.length - 1];
                fetch('/sync/commands?command_id=restart-proof%3A1&card_id=1', { method: 'POST' })
                  .then(async (response) => done({ status: response.status, body: await response.json() }))
                  .catch((error) => done({ error: String(error) }));
                "#,
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(after_restart["status"], 200);
        assert_eq!(after_restart["body"], accepted["body"]);

        driver
            .execute(
                r#"
                window.__restartReplay = null;
                const source = new EventSource('/sync/acknowledgements?after=0');
                source.addEventListener('acknowledgement', (event) => {
                  window.__restartReplay = { id: event.lastEventId, body: JSON.parse(event.data) };
                  source.close();
                });
                return true;
                "#,
                Vec::new(),
            )
            .await?;
        wait_until(&driver, "return window.__restartReplay !== null").await?;
        let replay = driver
            .execute("return window.__restartReplay", Vec::new())
            .await?
            .json()
            .clone();
        assert_eq!(replay["id"], "1");
        assert_eq!(replay["body"], accepted["body"]);

        driver.refresh().await?;
        let canonical = driver
            .execute(
                "return [...document.querySelectorAll('section.column')].map((column) => ({ title: column.querySelector('h2').textContent, cards: [...column.querySelectorAll('[data-key]')].map((card) => card.dataset.key) }))",
                Vec::new(),
            )
            .await?
            .json()
            .clone();
        assert_eq!(canonical[2]["title"], "Done");
        assert_eq!(canonical[2]["cards"], serde_json::json!(["1", "3"]));
        drop(second_app);
        Ok(())
    }
    .await;
    let quit = driver.quit().await;
    let _ = fs::remove_file(&store);
    result.and(quit)
}

async fn command_count(driver: &WebDriver) -> WebDriverResult<u64> {
    let count = driver
        .execute_async(
            r#"
            const done = arguments[arguments.length - 1];
            const open = indexedDB.open('hemx-kanban-v1', 1);
            open.onsuccess = () => {
              const request = open.result.transaction('commands', 'readonly').objectStore('commands').count();
              request.onsuccess = () => done(request.result);
              request.onerror = () => done({ error: request.error && request.error.name });
            };
            "#,
            Vec::new(),
        )
        .await?
        .json()
        .clone();
    Ok(count.as_u64().expect("durable command count"))
}

async fn wait_until(driver: &WebDriver, script: &str) -> WebDriverResult<()> {
    for _ in 0..200 {
        if driver.execute(script, Vec::new()).await?.json().as_bool() == Some(true) {
            return Ok(());
        }
        tokio::time::sleep(Duration::from_millis(25)).await;
    }
    let snapshot = driver
        .execute(
            "const sync = document.querySelector('[data-kanban-sync]'); return { url: location.href, body: document.body.textContent, sync: sync ? Object.fromEntries([...sync.attributes].map((attribute) => [attribute.name, attribute.value])) : null }",
            Vec::new(),
        )
        .await?
        .json()
        .clone();
    panic!("browser condition timed out: {script}; snapshot: {snapshot}");
}

fn available_port() -> u16 {
    TcpListener::bind("127.0.0.1:0")
        .expect("reserve browser test port")
        .local_addr()
        .expect("browser test address")
        .port()
}