feat(kanban): persist canonical acknowledgements

req: sync/001\nreq: sync/005\nreq: sync/007\nreq: sync/008\nreq: sync/013

examples/kanban/tests/browser_e2e.rs

@@ -1,4 +1,5 @@
 use hemx_test::TestProcess;
+use std::fs;
 use std::net::TcpListener;
 use std::process::Command;
 use std::time::Duration;
@@ -368,6 +369,134 @@ async fn pending_local_command_uploads_with_bounded_retry_and_is_removed_on_ack(
     result.and(quit)
 }
 
+#[tokio::test]
+async fn canonical_acknowledgement_survives_server_restart() -> WebDriverResult<()> {
+    // test req: sync/001 req: sync/005 req: sync/007 req: sync/008 req: sync/013
+    let app_port = available_port();
+    let app_addr = format!("127.0.0.1:{app_port}");
+    let store = std::env::temp_dir().join(format!(
+        "hemx-kanban-sync-{}-{app_port}.json",
+        std::process::id()
+    ));
+    let _ = fs::remove_file(&store);
+
+    let mut first_app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
+    first_app_command
+        .env("HEMX_KANBAN_ADDR", &app_addr)
+        .env("HEMX_KANBAN_SYNC_STORE", &store);
+    let first_app = TestProcess::start(
+        first_app_command,
+        "hemx-kanban-first",
+        &app_addr,
+        STARTUP_TIMEOUT,
+    )
+    .expect("start first hemx-kanban");
+
+    let webdriver_port = available_port();
+    let webdriver_addr = format!("127.0.0.1:{webdriver_port}");
+    let mut webdriver = Command::new("geckodriver");
+    webdriver.arg("--port").arg(webdriver_port.to_string());
+    let _webdriver = TestProcess::start(webdriver, "geckodriver", &webdriver_addr, STARTUP_TIMEOUT)
+        .expect("start ready geckodriver");
+    let mut caps = DesiredCapabilities::firefox();
+    caps.set_headless()?;
+    let driver = WebDriver::new(&format!("http://{webdriver_addr}"), caps).await?;
+
+    let result = async {
+        driver.goto(&format!("http://{app_addr}/")).await?;
+        let accepted = driver
+            .execute_async(
+                r#"
+                const done = arguments[arguments.length - 1];
+                fetch('/sync/commands?command_id=restart-proof%3A1&card_id=1', { method: 'POST' })
+                  .then(async (response) => done({ status: response.status, body: await response.json() }))
+                  .catch((error) => done({ error: String(error) }));
+                "#,
+                Vec::new(),
+            )
+            .await?
+            .json()
+            .clone();
+        assert_eq!(accepted["status"], 200);
+        assert_eq!(accepted["body"]["serverSequence"], 1);
+        assert_eq!(accepted["body"]["canonicalColumn"], "done");
+        assert!(store.is_file(), "server did not materialize sync store");
+        let persisted = fs::read_to_string(&store).expect("read sync store");
+        assert!(persisted.contains("restart-proof:1"));
+        assert!(persisted.contains("\"schemaVersion\": 1"));
+
+        drop(first_app);
+        let mut second_app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
+        second_app_command
+            .env("HEMX_KANBAN_ADDR", &app_addr)
+            .env("HEMX_KANBAN_SYNC_STORE", &store);
+        let second_app = TestProcess::start(
+            second_app_command,
+            "hemx-kanban-second",
+            &app_addr,
+            STARTUP_TIMEOUT,
+        )
+        .expect("restart hemx-kanban from durable sync store");
+
+        let after_restart = driver
+            .execute_async(
+                r#"
+                const done = arguments[arguments.length - 1];
+                fetch('/sync/commands?command_id=restart-proof%3A1&card_id=1', { method: 'POST' })
+                  .then(async (response) => done({ status: response.status, body: await response.json() }))
+                  .catch((error) => done({ error: String(error) }));
+                "#,
+                Vec::new(),
+            )
+            .await?
+            .json()
+            .clone();
+        assert_eq!(after_restart["status"], 200);
+        assert_eq!(after_restart["body"], accepted["body"]);
+
+        driver
+            .execute(
+                r#"
+                window.__restartReplay = null;
+                const source = new EventSource('/sync/acknowledgements?after=0');
+                source.addEventListener('acknowledgement', (event) => {
+                  window.__restartReplay = { id: event.lastEventId, body: JSON.parse(event.data) };
+                  source.close();
+                });
+                return true;
+                "#,
+                Vec::new(),
+            )
+            .await?;
+        wait_until(&driver, "return window.__restartReplay !== null").await?;
+        let replay = driver
+            .execute("return window.__restartReplay", Vec::new())
+            .await?
+            .json()
+            .clone();
+        assert_eq!(replay["id"], "1");
+        assert_eq!(replay["body"], accepted["body"]);
+
+        driver.refresh().await?;
+        let canonical = driver
+            .execute(
+                "return [...document.querySelectorAll('section.column')].map((column) => ({ title: column.querySelector('h2').textContent, cards: [...column.querySelectorAll('[data-key]')].map((card) => card.dataset.key) }))",
+                Vec::new(),
+            )
+            .await?
+            .json()
+            .clone();
+        assert_eq!(canonical[2]["title"], "Done");
+        assert_eq!(canonical[2]["cards"], serde_json::json!(["1", "3"]));
+        drop(second_app);
+        Ok(())
+    }
+    .await;
+    let quit = driver.quit().await;
+    let _ = fs::remove_file(&store);
+    result.and(quit)
+}
+
 async fn wait_until(driver: &WebDriver, script: &str) -> WebDriverResult<()> {
     for _ in 0..200 {
         if driver.execute(script, Vec::new()).await?.json().as_bool() == Some(true) {