fix(sync): bound acknowledgement stream

req: operations/004

examples/kanban/src/main.rs

@@ -25,6 +25,13 @@ use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 const COLUMNS: [(&str, &str); 3] = [("backlog", "Backlog"), ("doing", "Doing"), ("done", "Done")];
+const ACKNOWLEDGEMENT_STREAM_BUFFER_LIMIT: usize = 64;
+const ACKNOWLEDGEMENT_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15);
+const ACKNOWLEDGEMENT_RECONNECT_BACKOFF: [Duration; 3] = [
+    Duration::from_millis(100),
+    Duration::from_millis(250),
+    Duration::from_millis(500),
+];
 
 #[derive(Default)]
 struct AppState {
@@ -32,6 +39,7 @@ struct AppState {
     sync: Mutex<SyncState>,
     sync_store: Option<SyncStore>,
     sync_sessions: SyncSessionTokens,
+    acknowledgement_heartbeat_interval: Duration,
 }
 
 #[derive(Default)]
@@ -437,6 +445,12 @@ async fn main() {
         sync: Mutex::new(sync),
         sync_store,
         sync_sessions: SyncSessionTokens::from_env(),
+        acknowledgement_heartbeat_interval: std::env::var("HEMX_KANBAN_ACK_HEARTBEAT_MS")
+            .ok()
+            .and_then(|value| value.parse::<u64>().ok())
+            .filter(|milliseconds| *milliseconds > 0)
+            .map(Duration::from_millis)
+            .unwrap_or(ACKNOWLEDGEMENT_HEARTBEAT_INTERVAL),
     });
 
     let app = Router::new()
@@ -789,14 +803,17 @@ async fn sync_acknowledgements(
     if let Some(key) = reconnect_key {
         let attempts = sync.reconnects.entry(key).or_default();
         *attempts += 1;
-        if *attempts == 1 {
+        if let Some(backoff) = usize::try_from(*attempts - 1)
+            .ok()
+            .and_then(|index| ACKNOWLEDGEMENT_RECONNECT_BACKOFF.get(index))
+        {
             return Ok(Sse::new(
                 stream::iter([Ok(Event::default()
-                    .comment("reconnect")
-                    .retry(Duration::from_millis(25)))])
+                    .comment(format!("reconnect-attempt-{attempts}"))
+                    .retry(*backoff))])
                 .boxed(),
             )
-            .keep_alive(KeepAlive::default()));
+            .keep_alive(KeepAlive::new().interval(state.acknowledgement_heartbeat_interval)));
         }
     }
     let first_available = sync
@@ -817,7 +834,17 @@ async fn sync_acknowledgements(
         .unwrap_or_default();
     let history_missing = after < latest
         && first_available.is_none_or(|first_sequence| after.saturating_add(1) < first_sequence);
-    let events = if history_missing {
+    let pending_count = sync
+        .acknowledgements
+        .values()
+        .filter(|acknowledgement| {
+            visible_acknowledgement(principal, acknowledgement)
+                && acknowledgement.server_sequence > after
+                && acknowledgement.server_sequence > sync.retained_after
+        })
+        .count();
+    let slow_consumer = pending_count > ACKNOWLEDGEMENT_STREAM_BUFFER_LIMIT;
+    let events = if history_missing || slow_consumer {
         vec![Ok(Event::default()
             .id(latest.to_string())
             .event("snapshot-required")
@@ -826,6 +853,9 @@ async fn sync_acknowledgements(
                 "firstAvailable": first_available,
                 "latest": latest,
                 "snapshotUrl": "/sync/snapshot",
+                "reason": if slow_consumer { "slow-consumer" } else { "missing-history" },
+                "pendingCount": pending_count,
+                "bufferLimit": ACKNOWLEDGEMENT_STREAM_BUFFER_LIMIT,
             }))
             .expect("serializable missing history event"))]
     } else {
@@ -845,7 +875,24 @@ async fn sync_acknowledgements(
             })
             .collect::<Vec<Result<Event, Infallible>>>()
     };
-    Ok(Sse::new(stream::iter(events).boxed()).keep_alive(KeepAlive::default()))
+    drop(sync);
+    let heartbeat_interval = state.acknowledgement_heartbeat_interval;
+    let heartbeat = stream::unfold(heartbeat_interval, |interval| async move {
+        tokio::time::sleep(interval).await;
+        Some((
+            Ok(Event::default()
+                .event("heartbeat")
+                .data("{\"status\":\"ok\"}")),
+            interval,
+        ))
+    });
+    Ok(
+        Sse::new(stream::iter(events).chain(heartbeat).boxed()).keep_alive(
+            KeepAlive::new()
+                .interval(heartbeat_interval)
+                .text("heartbeat"),
+        ),
+    )
 }
 
 fn registry(state: Arc<AppState>) -> impl DispatchRegistry {