feat(kanban): recover after retry exhaustion

req: sync/004\nreq: sync/010\nreq: sync/011\nreq: sync/014\nreq: sync/016

examples/kanban/tests/browser_e2e.rs

@@ -369,6 +369,131 @@ async fn pending_local_command_uploads_with_bounded_retry_and_is_removed_on_ack(
     result.and(quit)
 }
 
+#[tokio::test]
+async fn exhausted_offline_retries_keep_command_until_later_reconnect() -> WebDriverResult<()> {
+    // test req: sync/004 req: sync/010 req: sync/011 req: sync/014 req: sync/016
+    let app_port = available_port();
+    let app_addr = format!("127.0.0.1:{app_port}");
+    let mut app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
+    app_command
+        .env("HEMX_KANBAN_ADDR", &app_addr)
+        .env("HEMX_KANBAN_SYNC_FAILURES", "3");
+    let _app = TestProcess::start(app_command, "hemx-kanban", &app_addr, STARTUP_TIMEOUT)
+        .expect("start ready hemx-kanban");
+
+    let webdriver_port = available_port();
+    let webdriver_addr = format!("127.0.0.1:{webdriver_port}");
+    let mut webdriver = Command::new("geckodriver");
+    webdriver.arg("--port").arg(webdriver_port.to_string());
+    let _webdriver = TestProcess::start(webdriver, "geckodriver", &webdriver_addr, STARTUP_TIMEOUT)
+        .expect("start ready geckodriver");
+    let mut caps = DesiredCapabilities::firefox();
+    caps.set_headless()?;
+    let driver = WebDriver::new(&format!("http://{webdriver_addr}"), caps).await?;
+
+    let result = async {
+        driver.goto(&format!("http://{app_addr}/")).await?;
+        let seeded = driver
+            .execute_async(
+                r#"
+                const done = arguments[arguments.length - 1];
+                const open = indexedDB.open('hemx-kanban-v1', 1);
+                open.onupgradeneeded = () => {
+                  const database = open.result;
+                  if (!database.objectStoreNames.contains('commands')) database.createObjectStore('commands', { keyPath: 'id' });
+                  if (!database.objectStoreNames.contains('meta')) database.createObjectStore('meta');
+                };
+                open.onsuccess = () => {
+                  const tx = open.result.transaction('commands', 'readwrite');
+                  tx.objectStore('commands').add({
+                    id: 'offline-actor:1', schemaVersion: 1, actor: 'offline-actor', session: 'offline-session',
+                    causal: 1, kind: 'reorder_card', cardId: '2', eventKind: 'click', key: null,
+                  });
+                  tx.oncomplete = () => done({ seeded: true });
+                  tx.onabort = () => done({ error: tx.error && tx.error.name });
+                };
+                "#,
+                Vec::new(),
+            )
+            .await?
+            .json()
+            .clone();
+        assert_eq!(seeded["seeded"], true, "failed to seed offline command: {seeded}");
+
+        driver.goto(&format!("http://{app_addr}/sync-demo")).await?;
+        wait_until(
+            &driver,
+            "return document.querySelector('[data-kanban-sync]')?.getAttribute('data-sync-phase') === 'offline'",
+        )
+        .await?;
+        let exhausted = driver
+            .execute(
+                "const root = document.querySelector('[data-kanban-sync]'); return { phase: root.getAttribute('data-sync-phase'), connection: root.getAttribute('data-sync-connection'), pending: root.getAttribute('data-sync-pending-count'), attempts: root.getAttribute('data-sync-attempts'), maxAttempts: root.getAttribute('data-sync-max-attempts'), manual: root.getAttribute('data-sync-manual-retry'), error: root.getAttribute('data-sync-error'), status: root.querySelector('[role=status]').textContent, retry: root.querySelector('[data-sync-retry]').textContent }",
+                Vec::new(),
+            )
+            .await?
+            .json()
+            .clone();
+        assert_eq!(exhausted["phase"], "offline");
+        assert_eq!(exhausted["connection"], "offline");
+        assert_eq!(exhausted["pending"], "1");
+        assert_eq!(exhausted["attempts"], "3");
+        assert_eq!(exhausted["maxAttempts"], "3");
+        assert_eq!(exhausted["manual"], "available");
+        assert_eq!(exhausted["error"], "sync upload failed with 503");
+        assert_eq!(
+            exhausted["status"],
+            "Sync is offline after bounded retries; the durable command remains queued. Retry now when ready."
+        );
+        assert_eq!(exhausted["retry"], "Retry sync now");
+        let queued = command_count(&driver).await?;
+        assert_eq!(queued, 1);
+
+        driver
+            .find(By::Css("[data-sync-retry]"))
+            .await?
+            .click()
+            .await?;
+        wait_until(
+            &driver,
+            "return document.querySelector('[data-kanban-sync]')?.getAttribute('data-sync-phase') === 'acknowledged'",
+        )
+        .await?;
+        let converged = driver
+            .execute(
+                "const root = document.querySelector('[data-kanban-sync]'); return { connection: root.getAttribute('data-sync-connection'), pending: root.getAttribute('data-sync-pending-count'), attempts: root.getAttribute('data-sync-attempts'), sequence: root.getAttribute('data-sync-ack-sequence'), column: root.getAttribute('data-sync-canonical-column'), status: root.querySelector('[role=status]').textContent, error: root.getAttribute('data-sync-error') }",
+                Vec::new(),
+            )
+            .await?
+            .json()
+            .clone();
+        assert_eq!(converged["connection"], "online");
+        assert_eq!(converged["pending"], "0");
+        assert_eq!(converged["attempts"], "1");
+        assert_eq!(converged["sequence"], "1");
+        assert_eq!(converged["column"], "done");
+        assert_eq!(converged["status"], "Command offline-actor:1 acknowledged in done.");
+        assert!(converged["error"].is_null());
+        assert_eq!(command_count(&driver).await?, 0);
+
+        driver.goto(&format!("http://{app_addr}/")).await?;
+        let canonical = driver
+            .execute(
+                "return [...document.querySelectorAll('section.column')].map((column) => ({ title: column.querySelector('h2').textContent, cards: [...column.querySelectorAll('[data-key]')].map((card) => card.dataset.key) }))",
+                Vec::new(),
+            )
+            .await?
+            .json()
+            .clone();
+        assert_eq!(canonical[2]["title"], "Done");
+        assert_eq!(canonical[2]["cards"], serde_json::json!(["2", "3"]));
+        Ok(())
+    }
+    .await;
+    let quit = driver.quit().await;
+    result.and(quit)
+}
+
 #[tokio::test]
 async fn canonical_acknowledgement_survives_server_restart() -> WebDriverResult<()> {
     // test req: sync/001 req: sync/005 req: sync/007 req: sync/008 req: sync/013
@@ -497,6 +622,26 @@ async fn canonical_acknowledgement_survives_server_restart() -> WebDriverResult<
     result.and(quit)
 }
 
+async fn command_count(driver: &WebDriver) -> WebDriverResult<u64> {
+    let count = driver
+        .execute_async(
+            r#"
+            const done = arguments[arguments.length - 1];
+            const open = indexedDB.open('hemx-kanban-v1', 1);
+            open.onsuccess = () => {
+              const request = open.result.transaction('commands', 'readonly').objectStore('commands').count();
+              request.onsuccess = () => done(request.result);
+              request.onerror = () => done({ error: request.error && request.error.name });
+            };
+            "#,
+            Vec::new(),
+        )
+        .await?
+        .json()
+        .clone();
+    Ok(count.as_u64().expect("durable command count"))
+}
+
 async fn wait_until(driver: &WebDriver, script: &str) -> WebDriverResult<()> {
     for _ in 0..200 {
         if driver.execute(script, Vec::new()).await?.json().as_bool() == Some(true) {