fix(sync): bound request lifetime
req: operations/003
This commit is contained in:
@@ -7,6 +7,7 @@ const LEGACY_COMMAND_SCHEMA = 1;
|
|||||||
const MIGRATION_KEY = "commandSchemaMigration";
|
const MIGRATION_KEY = "commandSchemaMigration";
|
||||||
const MAX_ATTEMPTS = 3;
|
const MAX_ATTEMPTS = 3;
|
||||||
const BACKOFF_MS = [25, 50];
|
const BACKOFF_MS = [25, 50];
|
||||||
|
const REQUEST_TIMEOUT_MS = 1_000;
|
||||||
const root = document.querySelector("[data-kanban-sync]");
|
const root = document.querySelector("[data-kanban-sync]");
|
||||||
const TAB_ID = sessionStorage.getItem("hemx-kanban-sync-tab-id") || crypto.randomUUID();
|
const TAB_ID = sessionStorage.getItem("hemx-kanban-sync-tab-id") || crypto.randomUUID();
|
||||||
const LEASE_MS = 5000;
|
const LEASE_MS = 5000;
|
||||||
@@ -16,6 +17,8 @@ let accountPartition;
|
|||||||
let uploadLimit;
|
let uploadLimit;
|
||||||
let retryTimer;
|
let retryTimer;
|
||||||
let leaseTimer;
|
let leaseTimer;
|
||||||
|
let acknowledgementSource;
|
||||||
|
const activeRequests = new Set();
|
||||||
let synchronizing = false;
|
let synchronizing = false;
|
||||||
let uploadsThisRun = 0;
|
let uploadsThisRun = 0;
|
||||||
let uploadedTotal = 0;
|
let uploadedTotal = 0;
|
||||||
@@ -39,6 +42,30 @@ class UploadError extends Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// req: operations/003
|
||||||
|
export async function fetchWithTimeout(
|
||||||
|
input,
|
||||||
|
init = {},
|
||||||
|
fetchImplementation = fetch,
|
||||||
|
timeoutMs = REQUEST_TIMEOUT_MS,
|
||||||
|
) {
|
||||||
|
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1) {
|
||||||
|
throw new TypeError("sync request timeout must be a positive integer");
|
||||||
|
}
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timeout = setTimeout(
|
||||||
|
() => controller.abort(new DOMException(`sync request timed out after ${timeoutMs} ms`, "TimeoutError")),
|
||||||
|
timeoutMs,
|
||||||
|
);
|
||||||
|
activeRequests.add(controller);
|
||||||
|
try {
|
||||||
|
return await fetchImplementation(input, { ...init, signal: controller.signal });
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
activeRequests.delete(controller);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function requestResult(request) {
|
function requestResult(request) {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
request.addEventListener("success", () => resolve(request.result), { once: true });
|
request.addEventListener("success", () => resolve(request.result), { once: true });
|
||||||
@@ -322,7 +349,7 @@ async function upload(command) {
|
|||||||
setPhase(attempt === 1 ? "uploading" : "retrying", `Uploading ${command.id} (attempt ${attempt} of ${MAX_ATTEMPTS}).`);
|
setPhase(attempt === 1 ? "uploading" : "retrying", `Uploading ${command.id} (attempt ${attempt} of ${MAX_ATTEMPTS}).`);
|
||||||
try {
|
try {
|
||||||
const query = new URLSearchParams({ command_id: command.id, card_id: command.cardId, column: command.targetColumn });
|
const query = new URLSearchParams({ command_id: command.id, card_id: command.cardId, column: command.targetColumn });
|
||||||
const response = await fetch(`/sync/commands?${query}`, { method: "POST" });
|
const response = await fetchWithTimeout(`/sync/commands?${query}`, { method: "POST" });
|
||||||
if (response.status === 503 && attempt < MAX_ATTEMPTS) {
|
if (response.status === 503 && attempt < MAX_ATTEMPTS) {
|
||||||
const base = BACKOFF_MS[attempt - 1];
|
const base = BACKOFF_MS[attempt - 1];
|
||||||
const delay = base + Math.floor(Math.random() * base);
|
const delay = base + Math.floor(Math.random() * base);
|
||||||
@@ -413,6 +440,7 @@ async function synchronize(command) {
|
|||||||
|
|
||||||
const reconnect = command.session || command.actor || "kanban";
|
const reconnect = command.session || command.actor || "kanban";
|
||||||
const source = new EventSource(`/sync/acknowledgements?after=0&reconnect=${encodeURIComponent(reconnect)}`);
|
const source = new EventSource(`/sync/acknowledgements?after=0&reconnect=${encodeURIComponent(reconnect)}`);
|
||||||
|
acknowledgementSource = source;
|
||||||
let opens = 0;
|
let opens = 0;
|
||||||
source.addEventListener("open", () => {
|
source.addEventListener("open", () => {
|
||||||
opens += 1;
|
opens += 1;
|
||||||
@@ -422,6 +450,7 @@ async function synchronize(command) {
|
|||||||
const canonical = JSON.parse(event.data);
|
const canonical = JSON.parse(event.data);
|
||||||
if (canonical.commandId !== command.id) return;
|
if (canonical.commandId !== command.id) return;
|
||||||
source.close();
|
source.close();
|
||||||
|
if (acknowledgementSource === source) acknowledgementSource = undefined;
|
||||||
root.setAttribute("data-sync-pending-before-ack", String((await pendingCommands(database)).length));
|
root.setAttribute("data-sync-pending-before-ack", String((await pendingCommands(database)).length));
|
||||||
const queueCommandId = command.queueCommandId || command.id;
|
const queueCommandId = command.queueCommandId || command.id;
|
||||||
await removePendingCommand(database, queueCommandId);
|
await removePendingCommand(database, queueCommandId);
|
||||||
@@ -454,7 +483,7 @@ async function synchronize(command) {
|
|||||||
});
|
});
|
||||||
source.addEventListener("snapshot-required", async (event) => {
|
source.addEventListener("snapshot-required", async (event) => {
|
||||||
const missing = JSON.parse(event.data);
|
const missing = JSON.parse(event.data);
|
||||||
const response = await fetch(missing.snapshotUrl);
|
const response = await fetchWithTimeout(missing.snapshotUrl);
|
||||||
if (!response.ok) throw new Error(`snapshot failed with ${response.status}`);
|
if (!response.ok) throw new Error(`snapshot failed with ${response.status}`);
|
||||||
const snapshot = await response.json();
|
const snapshot = await response.json();
|
||||||
const queued = await pendingCommands(database);
|
const queued = await pendingCommands(database);
|
||||||
@@ -497,6 +526,7 @@ async function synchronize(command) {
|
|||||||
}
|
}
|
||||||
synchronizing = false;
|
synchronizing = false;
|
||||||
source.close();
|
source.close();
|
||||||
|
if (acknowledgementSource === source) acknowledgementSource = undefined;
|
||||||
clearTimeout(leaseTimer);
|
clearTimeout(leaseTimer);
|
||||||
await releaseUploaderLease(database);
|
await releaseUploaderLease(database);
|
||||||
root.setAttribute("data-sync-leader", "false");
|
root.setAttribute("data-sync-leader", "false");
|
||||||
@@ -564,7 +594,8 @@ async function runLeaseLoop(command) {
|
|||||||
|
|
||||||
async function start() {
|
async function start() {
|
||||||
if (!root) return;
|
if (!root) return;
|
||||||
const contextResponse = await fetch("/sync/context", { credentials: "same-origin", cache: "no-store" });
|
root.setAttribute("data-sync-request-timeout-ms", String(REQUEST_TIMEOUT_MS));
|
||||||
|
const contextResponse = await fetchWithTimeout("/sync/context", { credentials: "same-origin", cache: "no-store" });
|
||||||
if (!contextResponse.ok) throw new Error(`account context failed with ${contextResponse.status}`);
|
if (!contextResponse.ok) throw new Error(`account context failed with ${contextResponse.status}`);
|
||||||
const context = await contextResponse.json();
|
const context = await contextResponse.json();
|
||||||
if (!context || typeof context.accountPartition !== "string" || !context.accountPartition) {
|
if (!context || typeof context.accountPartition !== "string" || !context.accountPartition) {
|
||||||
@@ -633,6 +664,12 @@ async function start() {
|
|||||||
window.addEventListener("pagehide", () => {
|
window.addEventListener("pagehide", () => {
|
||||||
stopped = true;
|
stopped = true;
|
||||||
clearTimeout(leaseTimer);
|
clearTimeout(leaseTimer);
|
||||||
|
clearTimeout(retryTimer);
|
||||||
|
acknowledgementSource?.close();
|
||||||
|
acknowledgementSource = undefined;
|
||||||
|
for (const controller of activeRequests) {
|
||||||
|
controller.abort(new DOMException("sync cancelled because page is hidden", "AbortError"));
|
||||||
|
}
|
||||||
if (database) releaseUploaderLease(database).catch(() => {});
|
if (database) releaseUploaderLease(database).catch(() => {});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -599,7 +599,7 @@ async fn account_partition_hides_replay_and_export_until_owner_returns() -> WebD
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn canonical_snapshot_and_history_are_tenant_scoped() -> WebDriverResult<()> {
|
async fn canonical_snapshot_and_history_are_tenant_scoped() -> WebDriverResult<()> {
|
||||||
// test req: sync/007 req: security/004 req: auth/005
|
// test req: sync/007 req: security/004 req: auth/005 req: performance/004
|
||||||
let app_port = available_port();
|
let app_port = available_port();
|
||||||
let app_addr = format!("127.0.0.1:{app_port}");
|
let app_addr = format!("127.0.0.1:{app_port}");
|
||||||
let mut app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
|
let mut app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
|
||||||
@@ -1708,7 +1708,7 @@ async fn missing_history_rebase_and_user_conflict_resolution_preserve_suffix() -
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn redacted_sync_diagnostics_are_bounded_and_leak_no_sensitive_material(
|
async fn redacted_sync_diagnostics_are_bounded_and_leak_no_sensitive_material(
|
||||||
) -> WebDriverResult<()> {
|
) -> WebDriverResult<()> {
|
||||||
// test req: operations/001 req: security/002 req: sync/016 req: sync/021
|
// test req: operations/001 req: operations/005 req: security/002 req: sync/016 req: sync/021
|
||||||
let app_port = available_port();
|
let app_port = available_port();
|
||||||
let app_addr = format!("127.0.0.1:{app_port}");
|
let app_addr = format!("127.0.0.1:{app_port}");
|
||||||
let mut app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
|
let mut app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
|
||||||
@@ -2077,6 +2077,81 @@ async fn keep_local_retry_preserves_conflicted_command_and_suffix_order() -> Web
|
|||||||
result.and(quit)
|
result.and(quit)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn sync_requests_timeout_and_cancel_on_pagehide() -> WebDriverResult<()> {
|
||||||
|
// test req: operations/003
|
||||||
|
let app_port = available_port();
|
||||||
|
let app_addr = format!("127.0.0.1:{app_port}");
|
||||||
|
let mut app_command = Command::new(env!("CARGO_BIN_EXE_hemx-kanban-example"));
|
||||||
|
app_command.env("HEMX_KANBAN_ADDR", &app_addr);
|
||||||
|
let _app = TestProcess::start(app_command, "hemx-kanban", &app_addr, STARTUP_TIMEOUT)
|
||||||
|
.expect("start hemx-kanban");
|
||||||
|
|
||||||
|
let webdriver_port = available_port();
|
||||||
|
let webdriver_addr = format!("127.0.0.1:{webdriver_port}");
|
||||||
|
let mut webdriver = Command::new("geckodriver");
|
||||||
|
webdriver.arg("--port").arg(webdriver_port.to_string());
|
||||||
|
let _webdriver = TestProcess::start(webdriver, "geckodriver", &webdriver_addr, STARTUP_TIMEOUT)
|
||||||
|
.expect("start ready geckodriver");
|
||||||
|
let mut caps = DesiredCapabilities::firefox();
|
||||||
|
caps.set_headless()?;
|
||||||
|
let driver = WebDriver::new(&format!("http://{webdriver_addr}"), caps).await?;
|
||||||
|
|
||||||
|
let result = async {
|
||||||
|
driver.goto(&format!("http://{app_addr}/sync-demo")).await?;
|
||||||
|
wait_until(
|
||||||
|
&driver,
|
||||||
|
"return document.querySelector('[data-kanban-sync]')?.getAttribute('data-sync-phase') === 'idle'",
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let proof = driver
|
||||||
|
.execute_async(
|
||||||
|
r#"
|
||||||
|
const done = arguments[arguments.length - 1];
|
||||||
|
(async () => {
|
||||||
|
const { fetchWithTimeout } = await import('/sync.js');
|
||||||
|
const pendingFetch = (_input, init) => new Promise((_resolve, reject) => {
|
||||||
|
init.signal.addEventListener('abort', () => reject(init.signal.reason), { once: true });
|
||||||
|
});
|
||||||
|
const started = performance.now();
|
||||||
|
let timeout;
|
||||||
|
try {
|
||||||
|
await fetchWithTimeout('/never-timeout', {}, pendingFetch, 40);
|
||||||
|
} catch (error) {
|
||||||
|
timeout = { name: error.name, message: error.message, elapsedMs: performance.now() - started };
|
||||||
|
}
|
||||||
|
const cancellationPromise = fetchWithTimeout('/never-pagehide', {}, pendingFetch, 10_000)
|
||||||
|
.then(() => ({ resolved: true }))
|
||||||
|
.catch((error) => ({ name: error.name, message: error.message }));
|
||||||
|
window.dispatchEvent(new PageTransitionEvent('pagehide'));
|
||||||
|
done({ timeout, cancellation: await cancellationPromise });
|
||||||
|
})().catch((error) => done({ error: String(error), stack: error?.stack }));
|
||||||
|
"#,
|
||||||
|
Vec::new(),
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
.json()
|
||||||
|
.clone();
|
||||||
|
assert!(proof["error"].is_null(), "bounded request failed: {proof}");
|
||||||
|
assert_eq!(proof["timeout"]["name"], "TimeoutError", "{proof}");
|
||||||
|
assert!(proof["timeout"]["message"]
|
||||||
|
.as_str()
|
||||||
|
.is_some_and(|message| message.contains("40 ms")));
|
||||||
|
assert!(proof["timeout"]["elapsedMs"]
|
||||||
|
.as_f64()
|
||||||
|
.is_some_and(|elapsed| (35.0..1_000.0).contains(&elapsed)));
|
||||||
|
assert_eq!(proof["cancellation"]["name"], "AbortError", "{proof}");
|
||||||
|
assert_eq!(
|
||||||
|
proof["cancellation"]["message"],
|
||||||
|
"sync cancelled because page is hidden"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
let quit = driver.quit().await;
|
||||||
|
result.and(quit)
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn identical_sync_inputs_reconcile_deterministically() -> WebDriverResult<()> {
|
async fn identical_sync_inputs_reconcile_deterministically() -> WebDriverResult<()> {
|
||||||
// test req: sync/022
|
// test req: sync/022
|
||||||
|
|||||||
Reference in New Issue
Block a user