test(sync): audit browser recovery paths
This commit is contained in:
+21
-16
@@ -842,6 +842,7 @@ async fn sync_acknowledgements(
|
|||||||
.get("reconnect")
|
.get("reconnect")
|
||||||
.filter(|value| !value.is_empty())
|
.filter(|value| !value.is_empty())
|
||||||
.cloned();
|
.cloned();
|
||||||
|
let persistent_stream = reconnect_key.is_some();
|
||||||
|
|
||||||
let mut sync = state.sync.lock().unwrap();
|
let mut sync = state.sync.lock().unwrap();
|
||||||
if let Some(key) = reconnect_key {
|
if let Some(key) = reconnect_key {
|
||||||
@@ -921,22 +922,26 @@ async fn sync_acknowledgements(
|
|||||||
};
|
};
|
||||||
drop(sync);
|
drop(sync);
|
||||||
let heartbeat_interval = state.acknowledgement_heartbeat_interval;
|
let heartbeat_interval = state.acknowledgement_heartbeat_interval;
|
||||||
let heartbeat = stream::unfold(heartbeat_interval, |interval| async move {
|
let event_stream = stream::iter(events).boxed();
|
||||||
tokio::time::sleep(interval).await;
|
let response_stream = if persistent_stream {
|
||||||
Some((
|
let heartbeat = stream::unfold(heartbeat_interval, |interval| async move {
|
||||||
Ok(Event::default()
|
tokio::time::sleep(interval).await;
|
||||||
.event("heartbeat")
|
Some((
|
||||||
.data("{\"status\":\"ok\"}")),
|
Ok(Event::default()
|
||||||
interval,
|
.event("heartbeat")
|
||||||
))
|
.data("{\"status\":\"ok\"}")),
|
||||||
});
|
interval,
|
||||||
Ok(
|
))
|
||||||
Sse::new(stream::iter(events).chain(heartbeat).boxed()).keep_alive(
|
});
|
||||||
KeepAlive::new()
|
event_stream.chain(heartbeat).boxed()
|
||||||
.interval(heartbeat_interval)
|
} else {
|
||||||
.text("heartbeat"),
|
event_stream
|
||||||
),
|
};
|
||||||
)
|
Ok(Sse::new(response_stream).keep_alive(
|
||||||
|
KeepAlive::new()
|
||||||
|
.interval(heartbeat_interval)
|
||||||
|
.text("heartbeat"),
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn registry(state: Arc<AppState>) -> impl DispatchRegistry {
|
fn registry(state: Arc<AppState>) -> impl DispatchRegistry {
|
||||||
|
|||||||
@@ -443,6 +443,21 @@ async function continuePendingWork() {
|
|||||||
setTimeout(() => synchronize(validatePending(commands[0])).catch(failPermanently), 0);
|
setTimeout(() => synchronize(validatePending(commands[0])).catch(failPermanently), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function renewOfflineLease(command) {
|
||||||
|
if (stopped || root.getAttribute("data-sync-phase") !== "offline") return;
|
||||||
|
const lease = await claimUploaderLease(database);
|
||||||
|
publishLease(lease);
|
||||||
|
if (!lease.leader) {
|
||||||
|
setPhase("standby", "Another tab owns sync; waiting for lease takeover.");
|
||||||
|
leaseTimer = setTimeout(() => runLeaseLoop(command).catch(failPermanently), LEASE_POLL_MS);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
leaseTimer = setTimeout(
|
||||||
|
() => renewOfflineLease(command).catch(failPermanently),
|
||||||
|
LEASE_MS / 2,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
async function synchronize(command) {
|
async function synchronize(command) {
|
||||||
if (synchronizing) return;
|
if (synchronizing) return;
|
||||||
synchronizing = true;
|
synchronizing = true;
|
||||||
@@ -455,7 +470,10 @@ async function synchronize(command) {
|
|||||||
}
|
}
|
||||||
clearTimeout(leaseTimer);
|
clearTimeout(leaseTimer);
|
||||||
leaseTimer = setTimeout(() => {
|
leaseTimer = setTimeout(() => {
|
||||||
if (!stopped && root.getAttribute("data-sync-phase") !== "acknowledged") {
|
if (stopped || root.getAttribute("data-sync-phase") === "acknowledged") return;
|
||||||
|
if (root.getAttribute("data-sync-phase") === "offline") {
|
||||||
|
renewOfflineLease(command).catch(failPermanently);
|
||||||
|
} else {
|
||||||
synchronize(command).catch(failPermanently);
|
synchronize(command).catch(failPermanently);
|
||||||
}
|
}
|
||||||
}, LEASE_MS / 2);
|
}, LEASE_MS / 2);
|
||||||
|
|||||||
@@ -829,12 +829,16 @@ async fn schema_upgrade_preserves_queued_order_and_local_intent() -> WebDriverRe
|
|||||||
])
|
])
|
||||||
);
|
);
|
||||||
|
|
||||||
driver.find(By::Css("[data-sync-retry]")).await?.click().await?;
|
for expected_pending in [2, 1] {
|
||||||
wait_until(
|
driver.find(By::Css("[data-sync-retry]")).await?.click().await?;
|
||||||
&driver,
|
wait_until(
|
||||||
"const root = document.querySelector('[data-kanban-sync]'); return root?.getAttribute('data-sync-phase') === 'backpressured' && root?.getAttribute('data-sync-pending-count') === '1'",
|
&driver,
|
||||||
)
|
&format!(
|
||||||
.await?;
|
"const root = document.querySelector('[data-kanban-sync]'); return root?.getAttribute('data-sync-phase') === 'offline' && root?.getAttribute('data-sync-pending-count') === '{expected_pending}'"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
driver.find(By::Css("[data-sync-retry]")).await?.click().await?;
|
driver.find(By::Css("[data-sync-retry]")).await?.click().await?;
|
||||||
wait_until(
|
wait_until(
|
||||||
&driver,
|
&driver,
|
||||||
|
|||||||
@@ -675,7 +675,7 @@ async fn kanban_command_export_delete_and_reset_are_recoverable() -> WebDriverRe
|
|||||||
let first_export = export_commands(&driver).await?.json().clone();
|
let first_export = export_commands(&driver).await?.json().clone();
|
||||||
assert_eq!(first_export["schemaVersion"], 1);
|
assert_eq!(first_export["schemaVersion"], 1);
|
||||||
assert_eq!(first_export["commands"].as_array().map(Vec::len), Some(1));
|
assert_eq!(first_export["commands"].as_array().map(Vec::len), Some(1));
|
||||||
assert_eq!(first_export["commands"][0]["schemaVersion"], 1);
|
assert_eq!(first_export["commands"][0]["schemaVersion"], 2);
|
||||||
assert_eq!(first_export["commands"][0]["kind"], "reorder_card");
|
assert_eq!(first_export["commands"][0]["kind"], "reorder_card");
|
||||||
assert_eq!(first_export["commands"][0]["cardId"], "1");
|
assert_eq!(first_export["commands"][0]["cardId"], "1");
|
||||||
let mut exported_keys = first_export["commands"][0]
|
let mut exported_keys = first_export["commands"][0]
|
||||||
@@ -688,6 +688,7 @@ async fn kanban_command_export_delete_and_reset_are_recoverable() -> WebDriverRe
|
|||||||
assert_eq!(
|
assert_eq!(
|
||||||
exported_keys,
|
exported_keys,
|
||||||
[
|
[
|
||||||
|
"accountPartition",
|
||||||
"actor",
|
"actor",
|
||||||
"cardId",
|
"cardId",
|
||||||
"causal",
|
"causal",
|
||||||
@@ -695,8 +696,10 @@ async fn kanban_command_export_delete_and_reset_are_recoverable() -> WebDriverRe
|
|||||||
"id",
|
"id",
|
||||||
"key",
|
"key",
|
||||||
"kind",
|
"kind",
|
||||||
|
"queuedAt",
|
||||||
"schemaVersion",
|
"schemaVersion",
|
||||||
"session",
|
"session",
|
||||||
|
"targetColumn",
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
let first_actor = first_export["commands"][0]["actor"]
|
let first_actor = first_export["commands"][0]["actor"]
|
||||||
@@ -1205,7 +1208,7 @@ async fn kanban_replay_is_bounded_and_within_budget() -> WebDriverResult<()> {
|
|||||||
assert_eq!(within_bound["order"], "2|1");
|
assert_eq!(within_bound["order"], "2|1");
|
||||||
assert_eq!(within_bound["limit"], REPLAY_LIMIT.to_string());
|
assert_eq!(within_bound["limit"], REPLAY_LIMIT.to_string());
|
||||||
assert_eq!(within_bound["budget"], REPLAY_BUDGET_MS);
|
assert_eq!(within_bound["budget"], REPLAY_BUDGET_MS);
|
||||||
assert_eq!(within_bound["over"], false);
|
assert_eq!(within_bound["over"], false, "{within_bound}");
|
||||||
assert!(
|
assert!(
|
||||||
within_bound["elapsed"].as_f64().is_some_and(|elapsed| elapsed <= REPLAY_BUDGET_MS),
|
within_bound["elapsed"].as_f64().is_some_and(|elapsed| elapsed <= REPLAY_BUDGET_MS),
|
||||||
"replay exceeded budget: {within_bound}"
|
"replay exceeded budget: {within_bound}"
|
||||||
@@ -1884,12 +1887,15 @@ async fn store_replay_commands(driver: &WebDriver, first: u64, last: u64) -> Web
|
|||||||
for (let causal = first; causal <= last; causal += 1) {
|
for (let causal = first; causal <= last; causal += 1) {
|
||||||
commands.add({
|
commands.add({
|
||||||
id: `replay:${causal}`,
|
id: `replay:${causal}`,
|
||||||
schemaVersion: 1,
|
schemaVersion: 2,
|
||||||
|
accountPartition: 'demo:demo',
|
||||||
actor: 'replay',
|
actor: 'replay',
|
||||||
session: 'replay',
|
session: 'replay',
|
||||||
causal,
|
causal,
|
||||||
|
queuedAt: Date.now(),
|
||||||
kind: 'reorder_card',
|
kind: 'reorder_card',
|
||||||
cardId: '1',
|
cardId: '1',
|
||||||
|
targetColumn: 'done',
|
||||||
eventKind: 'click',
|
eventKind: 'click',
|
||||||
key: null,
|
key: null,
|
||||||
});
|
});
|
||||||
@@ -1922,12 +1928,15 @@ async fn store_malformed_command(driver: &WebDriver) -> WebDriverResult<()> {
|
|||||||
const tx = open.result.transaction('commands', 'readwrite');
|
const tx = open.result.transaction('commands', 'readwrite');
|
||||||
tx.objectStore('commands').add({
|
tx.objectStore('commands').add({
|
||||||
id: 'corrupt:1',
|
id: 'corrupt:1',
|
||||||
schemaVersion: 1,
|
schemaVersion: 2,
|
||||||
|
accountPartition: 'demo:demo',
|
||||||
actor: 'corrupt',
|
actor: 'corrupt',
|
||||||
session: 'corrupt',
|
session: 'corrupt',
|
||||||
causal: 1,
|
causal: 1,
|
||||||
|
queuedAt: Date.now(),
|
||||||
kind: 'reorder_card',
|
kind: 'reorder_card',
|
||||||
cardId: '',
|
cardId: '',
|
||||||
|
targetColumn: 'done',
|
||||||
eventKind: 'click',
|
eventKind: 'click',
|
||||||
key: null,
|
key: null,
|
||||||
});
|
});
|
||||||
@@ -1967,19 +1976,22 @@ async fn occupy_next_command_id(driver: &WebDriver) -> WebDriverResult<()> {
|
|||||||
const next = causal + 1;
|
const next = causal + 1;
|
||||||
commands.add({
|
commands.add({
|
||||||
id: `${actor}:${next}`,
|
id: `${actor}:${next}`,
|
||||||
schemaVersion: 1,
|
schemaVersion: 2,
|
||||||
|
accountPartition: 'demo:demo',
|
||||||
actor,
|
actor,
|
||||||
session: 'fault-injection',
|
session: 'fault-injection',
|
||||||
causal: next,
|
causal: next,
|
||||||
|
queuedAt: Date.now(),
|
||||||
kind: 'reorder_card',
|
kind: 'reorder_card',
|
||||||
cardId: 'fault-injection',
|
cardId: 'fault-injection',
|
||||||
|
targetColumn: 'done',
|
||||||
eventKind: 'click',
|
eventKind: 'click',
|
||||||
key: null,
|
key: null,
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
const actorRequest = meta.get('actor');
|
const actorRequest = meta.get('actor:demo:demo');
|
||||||
actorRequest.onsuccess = () => { actor = actorRequest.result; addCollision(); };
|
actorRequest.onsuccess = () => { actor = actorRequest.result; addCollision(); };
|
||||||
const causalRequest = meta.get('causal');
|
const causalRequest = meta.get('causal:demo:demo');
|
||||||
causalRequest.onsuccess = () => { causal = causalRequest.result; addCollision(); };
|
causalRequest.onsuccess = () => { causal = causalRequest.result; addCollision(); };
|
||||||
tx.oncomplete = () => done({ id: `${actor}:${causal + 1}` });
|
tx.oncomplete = () => done({ id: `${actor}:${causal + 1}` });
|
||||||
tx.onabort = () => done({ error: tx.error && tx.error.name });
|
tx.onabort = () => done({ error: tx.error && tx.error.name });
|
||||||
|
|||||||
Reference in New Issue
Block a user