fix(sync): bound acknowledgement stream

req: operations/004

examples/kanban/tests/browser_e2e.rs

@@ -2196,6 +2196,113 @@ async fn adversarial_wire_inputs_are_rejected_before_partial_application() -> We
     result.and(quit)
 }
 
+#[tokio::test]
+async fn acknowledgement_stream_bounds_reconnect_buffering_heartbeat_and_cancellation(
+) -> WebDriverResult<()> {
+    // test req: operations/004
+    let app_port = available_port();
+    let app_addr = format!("127.0.0.1:{app_port}");
+    let mut app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
+    app_command
+        .env("HEMX_KANBAN_ADDR", &app_addr)
+        .env("HEMX_KANBAN_ACK_HEARTBEAT_MS", "25");
+    let _app = TestProcess::start(app_command, "hemx-kanban", &app_addr, STARTUP_TIMEOUT)
+        .expect("start hemx-kanban");
+
+    let webdriver_port = available_port();
+    let webdriver_addr = format!("127.0.0.1:{webdriver_port}");
+    let mut webdriver = Command::new("geckodriver");
+    webdriver.arg("--port").arg(webdriver_port.to_string());
+    let _webdriver = TestProcess::start(webdriver, "geckodriver", &webdriver_addr, STARTUP_TIMEOUT)
+        .expect("start ready geckodriver");
+    let mut caps = DesiredCapabilities::firefox();
+    caps.set_headless()?;
+    let driver = WebDriver::new(&format!("http://{webdriver_addr}"), caps).await?;
+
+    let result = async {
+        driver.goto(&format!("http://{app_addr}/sync-demo")).await?;
+        wait_until(
+            &driver,
+            "return document.querySelector('[data-kanban-sync]')?.getAttribute('data-sync-phase') === 'idle'",
+        )
+        .await?;
+        let proof = driver
+            .execute_async(
+                r#"
+                const done = arguments[arguments.length - 1];
+                (async () => {
+                  for (let index = 1; index <= 65; index += 1) {
+                    const query = new URLSearchParams({
+                      command_id: `buffer:${index}`,
+                      card_id: '1',
+                      column: 'done',
+                    });
+                    const response = await fetch(`/sync/commands?${query}`, { method: 'POST' });
+                    if (!response.ok) throw new Error(`command ${index} failed with ${response.status}`);
+                  }
+
+                  const observe = (url, eventName, timeoutMs = 5000) => new Promise((resolve, reject) => {
+                    const started = performance.now();
+                    const source = new EventSource(url);
+                    let opens = 0;
+                    let errors = 0;
+                    const timeout = setTimeout(() => {
+                      source.close();
+                      reject(new Error(`${eventName} timed out after ${timeoutMs} ms`));
+                    }, timeoutMs);
+                    source.addEventListener('open', () => { opens += 1; });
+                    source.addEventListener('error', () => { errors += 1; });
+                    source.addEventListener(eventName, (event) => {
+                      clearTimeout(timeout);
+                      const data = JSON.parse(event.data);
+                      source.close();
+                      resolve({
+                        data,
+                        opens,
+                        errors,
+                        elapsedMs: performance.now() - started,
+                        cancelled: source.readyState === EventSource.CLOSED,
+                      });
+                    });
+                  });
+
+                  const slowConsumer = await observe(
+                    '/sync/acknowledgements?after=0&reconnect=slow-consumer-proof',
+                    'snapshot-required',
+                  );
+                  const heartbeat = await observe(
+                    '/sync/acknowledgements?after=65&reconnect=heartbeat-proof',
+                    'heartbeat',
+                  );
+                  done({ slowConsumer, heartbeat });
+                })().catch((error) => done({ error: String(error), stack: error?.stack }));
+                "#,
+                Vec::new(),
+            )
+            .await?
+            .json()
+            .clone();
+        assert!(proof["error"].is_null(), "stream bounds failed: {proof}");
+        assert_eq!(proof["slowConsumer"]["data"]["reason"], "slow-consumer");
+        assert_eq!(proof["slowConsumer"]["data"]["pendingCount"], 65);
+        assert_eq!(proof["slowConsumer"]["data"]["bufferLimit"], 64);
+        assert_eq!(proof["slowConsumer"]["opens"], 4);
+        assert!(proof["slowConsumer"]["errors"].as_u64().is_some_and(|errors| errors >= 3));
+        assert!(proof["slowConsumer"]["elapsedMs"]
+            .as_f64()
+            .is_some_and(|elapsed| (700.0..5_000.0).contains(&elapsed)));
+        assert_eq!(proof["slowConsumer"]["cancelled"], true);
+        assert_eq!(proof["heartbeat"]["data"]["status"], "ok");
+        assert_eq!(proof["heartbeat"]["opens"], 4);
+        assert!(proof["heartbeat"]["errors"].as_u64().is_some_and(|errors| errors >= 3));
+        assert_eq!(proof["heartbeat"]["cancelled"], true);
+        Ok(())
+    }
+    .await;
+    let quit = driver.quit().await;
+    result.and(quit)
+}
+
 #[tokio::test]
 async fn sync_requests_timeout_and_cancel_on_pagehide() -> WebDriverResult<()> {
     // test req: operations/003