feat(kanban): recover after retry exhaustion

req: sync/004\nreq: sync/010\nreq: sync/011\nreq: sync/014\nreq: sync/016

examples/kanban/src/main.rs

@@ -15,7 +15,7 @@ use hemx_kanban_example::ui::board::{self as board};
 use hemx_kanban_example::ui::board_card as card_board;
 use hemx_kanban_example::ui::{self, board as board_ui};
 use serde::{Deserialize, Serialize};
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::BTreeMap;
 use std::convert::Infallible;
 use std::fs::{self, OpenOptions};
 use std::io::Write;
@@ -41,8 +41,8 @@ struct SyncState {
     next_sequence: u64,
     acknowledgements: BTreeMap<CommandId, SyncAcknowledgement>,
     reconnects: BTreeMap<String, u64>,
-    fail_first_upload: bool,
-    transient_failures: BTreeSet<CommandId>,
+    transient_failure_limit: u8,
+    transient_failures: BTreeMap<CommandId, u8>,
 }
 
 #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
@@ -294,7 +294,10 @@ async fn main() {
             next_sequence: 1,
             ..SyncState::default()
         });
-    sync.fail_first_upload = std::env::var_os("HEMX_KANBAN_FAIL_FIRST_SYNC").is_some();
+    sync.transient_failure_limit = std::env::var("HEMX_KANBAN_SYNC_FAILURES")
+        .ok()
+        .and_then(|value| value.parse::<u8>().ok())
+        .unwrap_or_else(|| u8::from(std::env::var_os("HEMX_KANBAN_FAIL_FIRST_SYNC").is_some()));
     let mut board = initial_board();
     for acknowledgement in sync.acknowledgements.values() {
         if let Some(card) = board
@@ -433,8 +436,16 @@ async fn sync_command(
         }
         return Ok(Json(existing.clone()));
     }
-    if sync.fail_first_upload && sync.transient_failures.insert(command_id.clone()) {
-        return Err(SyncRejection::Transient);
+    let transient_failure_limit = sync.transient_failure_limit;
+    if transient_failure_limit > 0 {
+        let failures = sync
+            .transient_failures
+            .entry(command_id.clone())
+            .or_default();
+        if *failures < transient_failure_limit {
+            *failures += 1;
+            return Err(SyncRejection::Transient);
+        }
     }
 
     let mut board = state.board.lock().unwrap();