feat(saas): correlate mutation diagnostics

req: operations/001

req: operations/003

req: security/008
This commit is contained in:
slhx agent
2026-07-14 00:53:04 +02:00
parent 4e0d9ebaff
commit 4fc5e53adf
4 changed files with 78 additions and 13 deletions
+3 -3
View File
@@ -60,13 +60,13 @@ encryption, retention, backup, and deployment policy remain host concerns.
## Slice 6 — production integration reference ## Slice 6 — production integration reference
- [ ] **User value:** adopters can copy a proven boundary for durable storage, auth, transactions, security controls, observability, and restart recovery without hemx owning vendor policy. - [ ] **User value:** adopters can copy a proven boundary for durable storage, auth, transactions, security controls, observability, and restart recovery without hemx owning vendor policy.
- **State:** In progress — the public server-first mutation now additionally exposes dependency-aware liveness/readiness, aggregate vendor-neutral metrics, typed secret-free diagnostics, and stale-build fail-closed reload recovery. - **State:** In progress — the requirements audit corrected stale namespace aliases and found one missing production-reference control: stable request/session/user correlation. The mutation now returns a generated request ID and emits typed request/session/user diagnostic correlation without accepting caller-controlled identifiers or secrets.
- **Build:** evolve one existing reference app using ordinary integration adapters; add durable app storage, authenticated/authorized allowed and denied mutations, CSRF/origin checks, transaction rollback, bounded input, structured failures, health/readiness, tracing/metrics hooks, and restart/deploy recovery. - **Build:** evolve one existing reference app using ordinary integration adapters; add durable app storage, authenticated/authorized allowed and denied mutations, CSRF/origin checks, transaction rollback, bounded input, structured failures, health/readiness, tracing/metrics hooks, and restart/deploy recovery.
- **Refusals:** no built-in database/auth provider, compliance claim, telemetry vendor, deployment system, or repository framework. - **Refusals:** no built-in database/auth provider, compliance claim, telemetry vendor, deployment system, or repository framework.
- **Requirements:** `security/001-009`, `operations/001-008`, `v1_release/003`, existing `adapter/*`, `integration/*`, and `diagnostics/*` contracts. - **Requirements:** `security/001-009`, `operations/001-008`, `v1_release/003`, and the existing `auth/*`, `axum/*`, `failure/*`, and `diag/*` contracts (the former `adapter/*`, `integration/*`, and `diagnostics/*` cursor names do not exist in `REQUIREMENTS.md`).
- **Proof:** end-to-end test survives process restart and mixed deployment, proves allowed/denied/rolled-back mutations and redacted diagnostics, and maps each framework-owned ASVS-relevant control to a failing/passing case. - **Proof:** end-to-end test survives process restart and mixed deployment, proves allowed/denied/rolled-back mutations and redacted diagnostics, and maps each framework-owned ASVS-relevant control to a failing/passing case.
`cargo test -p hemx-saas-example --test production_reference` proves current bearer authentication, origin/CSRF denial, bounded input, atomic durable commit, rollback, restart recovery, live-vs-ready dependency failure, redacted structured problem responses, aggregate metrics, and stale-fingerprint rejection followed by current-build recovery. The diagnostic hook accepts only typed outcome and duration fields, preventing request secrets from entering framework-owned diagnostic records. Execution cursor: audit Slice 6 requirement citations against `security/001-009`, `operations/001-008`, `adapter/*`, `integration/*`, and `diagnostics/*`; add only the missing end-to-end control cases, then close Slice 6 if clean. `cargo test -p hemx-saas-example --test production_reference` proves current bearer authentication, origin/CSRF denial, bounded input, atomic durable commit, rollback, restart recovery, live-vs-ready dependency failure, redacted structured problem responses, aggregate metrics, stale-fingerprint rejection followed by current-build recovery, and generated request correlation on allowed, denied, mismatch, and failed mutations. The typed diagnostic hook carries only generated request ID, fixed session/user IDs, outcome, and duration, preventing request secrets from entering framework-owned records. Execution cursor: finish the corrected `security/*`, `operations/*`, `auth/*`, `axum/*`, `failure/*`, and `diag/*` citation audit; add only any remaining missing end-to-end control, then close Slice 6 if clean.
## Slice 7 — v1 compatibility and closure ## Slice 7 — v1 compatibility and closure
+48 -5
View File
@@ -19,8 +19,12 @@ use std::time::Duration;
use ui::dashboard; use ui::dashboard;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct SessionId(u64);
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub struct Session { pub struct Session {
session_id: SessionId,
user_id: UserId, user_id: UserId,
email: String, email: String,
csrf: CsrfToken, csrf: CsrfToken,
@@ -31,6 +35,7 @@ pub struct Session {
impl Session { impl Session {
pub fn demo() -> Self { pub fn demo() -> Self {
Self { Self {
session_id: SessionId(1),
user_id: UserId(42), user_id: UserId(42),
email: "founder@example.com".to_owned(), email: "founder@example.com".to_owned(),
csrf: CsrfToken("demo-csrf".to_owned()), csrf: CsrfToken("demo-csrf".to_owned()),
@@ -204,8 +209,20 @@ fn persist_projects(path: &Path, projects: &[ProjectRecord]) -> io::Result<()> {
Ok(()) Ok(())
} }
#[derive(Clone, Copy)] #[derive(Clone, Debug, PartialEq, Eq)]
pub struct RequestCorrelationId(String);
impl Display for RequestCorrelationId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
#[derive(Clone)]
pub struct MutationDiagnostic { pub struct MutationDiagnostic {
pub request_id: RequestCorrelationId,
pub session_id: SessionId,
pub user_id: UserId,
pub outcome: &'static str, pub outcome: &'static str,
pub duration_micros: u64, pub duration_micros: u64,
} }
@@ -219,8 +236,12 @@ struct StderrDiagnosticSink;
impl DiagnosticSink for StderrDiagnosticSink { impl DiagnosticSink for StderrDiagnosticSink {
fn record(&self, diagnostic: MutationDiagnostic) { fn record(&self, diagnostic: MutationDiagnostic) {
eprintln!( eprintln!(
"event=saas.project_mutation outcome={} duration_micros={}", "event=saas.project_mutation request_id={} session_id={} user_id={} outcome={} duration_micros={}",
diagnostic.outcome, diagnostic.duration_micros diagnostic.request_id,
diagnostic.session_id.0,
diagnostic.user_id.0,
diagnostic.outcome,
diagnostic.duration_micros
); );
} }
} }
@@ -234,6 +255,7 @@ struct MutationMetrics {
mismatch: AtomicU64, mismatch: AtomicU64,
failed: AtomicU64, failed: AtomicU64,
duration_micros: AtomicU64, duration_micros: AtomicU64,
next_request_id: AtomicU64,
} }
#[derive(Clone)] #[derive(Clone)]
@@ -300,7 +322,21 @@ impl AppContext {
self self
} }
pub fn record_mutation(&self, outcome: &'static str, duration: Duration) { pub fn next_request_id(&self) -> RequestCorrelationId {
let sequence = self
.metrics
.next_request_id
.fetch_add(1, Ordering::Relaxed)
.saturating_add(1);
RequestCorrelationId(format!("req-{}-{sequence}", std::process::id()))
}
pub fn record_mutation(
&self,
request_id: RequestCorrelationId,
outcome: &'static str,
duration: Duration,
) {
self.metrics.attempts.fetch_add(1, Ordering::Relaxed); self.metrics.attempts.fetch_add(1, Ordering::Relaxed);
match outcome { match outcome {
"succeeded" => &self.metrics.succeeded, "succeeded" => &self.metrics.succeeded,
@@ -315,6 +351,9 @@ impl AppContext {
.duration_micros .duration_micros
.fetch_add(duration_micros, Ordering::Relaxed); .fetch_add(duration_micros, Ordering::Relaxed);
self.diagnostics.record(MutationDiagnostic { self.diagnostics.record(MutationDiagnostic {
request_id,
session_id: self.session.session_id,
user_id: self.session.user_id,
outcome, outcome,
duration_micros, duration_micros,
}); });
@@ -583,10 +622,14 @@ mod tests {
// req: operations/003 req: operations/005 req: security/008 // req: operations/003 req: operations/005 req: security/008
let diagnostics = Arc::new(RecordingDiagnostics::default()); let diagnostics = Arc::new(RecordingDiagnostics::default());
let ctx = AppContext::demo().with_diagnostic_sink(diagnostics.clone()); let ctx = AppContext::demo().with_diagnostic_sink(diagnostics.clone());
ctx.record_mutation("denied", Duration::from_micros(7)); let request_id = ctx.next_request_id();
ctx.record_mutation(request_id.clone(), "denied", Duration::from_micros(7));
let recorded = diagnostics.0.lock().unwrap(); let recorded = diagnostics.0.lock().unwrap();
assert_eq!(recorded.len(), 1); assert_eq!(recorded.len(), 1);
assert_eq!(recorded[0].request_id, request_id);
assert_eq!(recorded[0].session_id, SessionId(1));
assert_eq!(recorded[0].user_id, UserId(42));
assert_eq!(recorded[0].outcome, "denied"); assert_eq!(recorded[0].outcome, "denied");
assert_eq!(recorded[0].duration_micros, 7); assert_eq!(recorded[0].duration_micros, 7);
} }
+10 -4
View File
@@ -1,6 +1,6 @@
use axum::body::Body; use axum::body::Body;
use axum::extract::{DefaultBodyLimit, Form, Query, State}; use axum::extract::{DefaultBodyLimit, Form, Query, State};
use axum::http::{HeaderMap, StatusCode}; use axum::http::{HeaderMap, HeaderValue, StatusCode};
use axum::response::{IntoResponse, Response}; use axum::response::{IntoResponse, Response};
use axum::routing::{get, post}; use axum::routing::{get, post};
use axum::Router; use axum::Router;
@@ -80,6 +80,7 @@ async fn create_project(
Form(form): Form<BTreeMap<String, String>>, Form(form): Form<BTreeMap<String, String>>,
) -> Response { ) -> Response {
let started = Instant::now(); let started = Instant::now();
let request_id = ctx.next_request_id();
let bearer = headers let bearer = headers
.get("authorization") .get("authorization")
.and_then(|value| value.to_str().ok()) .and_then(|value| value.to_str().ok())
@@ -96,17 +97,18 @@ async fn create_project(
{ {
let current_fingerprint = ui::BUILD_FINGERPRINT.0.to_string(); let current_fingerprint = ui::BUILD_FINGERPRINT.0.to_string();
if client_fingerprint != current_fingerprint { if client_fingerprint != current_fingerprint {
ctx.record_mutation("mismatch", started.elapsed()); ctx.record_mutation(request_id.clone(), "mismatch", started.elapsed());
return Response::builder() return Response::builder()
.status(StatusCode::CONFLICT) .status(StatusCode::CONFLICT)
.header("content-type", "application/problem+json") .header("content-type", "application/problem+json")
.header("x-hemx-recovery", "reload") .header("x-hemx-recovery", "reload")
.header("x-hemx-fingerprint", current_fingerprint) .header("x-hemx-fingerprint", current_fingerprint)
.header("x-request-id", request_id.to_string())
.body(Body::from("{\"code\":\"deployment-mismatch\"}")) .body(Body::from("{\"code\":\"deployment-mismatch\"}"))
.expect("deployment mismatch response"); .expect("deployment mismatch response");
} }
} }
let (outcome, response) = match ctx.create_project_authorized(name, bearer, csrf, origin) { let (outcome, mut response) = match ctx.create_project_authorized(name, bearer, csrf, origin) {
Ok(_) => ( Ok(_) => (
"succeeded", "succeeded",
(StatusCode::SEE_OTHER, [("location", "/")], "").into_response(), (StatusCode::SEE_OTHER, [("location", "/")], "").into_response(),
@@ -128,7 +130,11 @@ async fn create_project(
problem(StatusCode::SERVICE_UNAVAILABLE, "storage-unavailable"), problem(StatusCode::SERVICE_UNAVAILABLE, "storage-unavailable"),
), ),
}; };
ctx.record_mutation(outcome, started.elapsed()); ctx.record_mutation(request_id.clone(), outcome, started.elapsed());
response.headers_mut().insert(
"x-request-id",
HeaderValue::from_str(&request_id.to_string()).expect("generated request ID is a header"),
);
response response
} }
+17 -1
View File
@@ -83,6 +83,16 @@ fn create_at_version(
) )
} }
fn response_header<'a>(response: &'a str, name: &str) -> &'a str {
response
.lines()
.find_map(|line| {
let (header_name, value) = line.split_once(':')?;
header_name.eq_ignore_ascii_case(name).then(|| value.trim())
})
.unwrap_or_else(|| panic!("missing {name} response header"))
}
fn ready_fingerprint(response: &str) -> &str { fn ready_fingerprint(response: &str) -> &str {
let marker = "\"fingerprint\":\""; let marker = "\"fingerprint\":\"";
let start = response.find(marker).expect("readiness fingerprint") + marker.len(); let start = response.find(marker).expect("readiness fingerprint") + marker.len();
@@ -92,7 +102,7 @@ fn ready_fingerprint(response: &str) -> &str {
#[test] #[test]
fn authenticated_project_mutation_is_atomic_and_survives_restart() { fn authenticated_project_mutation_is_atomic_and_survives_restart() {
// test req: auth/001 req: auth/002 req: auth/004 req: security/004 req: v1_release/003 // test req: auth/001 req: auth/002 req: auth/004 req: security/004 req: operations/001 req: v1_release/003
let address = available_address(); let address = available_address();
let origin = format!("http://{address}"); let origin = format!("http://{address}");
let store = test_path("durable"); let store = test_path("durable");
@@ -131,6 +141,7 @@ fn authenticated_project_mutation_is_atomic_and_survives_restart() {
]; ];
for denied in &denied_responses { for denied in &denied_responses {
assert!(denied.starts_with("HTTP/1.1 403"), "{denied}"); assert!(denied.starts_with("HTTP/1.1 403"), "{denied}");
assert!(response_header(denied, "x-request-id").starts_with("req-"));
assert!(denied.contains("{\"code\":\"authorization-denied\"}")); assert!(denied.contains("{\"code\":\"authorization-denied\"}"));
assert!(!denied.contains("Denied")); assert!(!denied.contains("Denied"));
assert!(!denied.contains("demo-csrf")); assert!(!denied.contains("demo-csrf"));
@@ -192,6 +203,7 @@ fn authenticated_project_mutation_is_atomic_and_survives_restart() {
Some("0"), Some("0"),
); );
assert!(stale.starts_with("HTTP/1.1 409"), "{stale}"); assert!(stale.starts_with("HTTP/1.1 409"), "{stale}");
assert!(response_header(&stale, "x-request-id").starts_with("req-"));
assert!(stale.contains("{\"code\":\"deployment-mismatch\"}")); assert!(stale.contains("{\"code\":\"deployment-mismatch\"}"));
assert!(stale assert!(stale
.to_ascii_lowercase() .to_ascii_lowercase()
@@ -208,6 +220,9 @@ fn authenticated_project_mutation_is_atomic_and_survives_restart() {
Some(fingerprint), Some(fingerprint),
); );
assert!(allowed.starts_with("HTTP/1.1 303"), "{allowed}"); assert!(allowed.starts_with("HTTP/1.1 303"), "{allowed}");
let allowed_request_id = response_header(&allowed, "x-request-id");
assert!(allowed_request_id.starts_with("req-"));
assert_ne!(allowed_request_id, response_header(&stale, "x-request-id"));
assert!(request(&address, "GET", "/", &[], "").contains("Durable Project")); assert!(request(&address, "GET", "/", &[], "").contains("Durable Project"));
let metrics = request(&address, "GET", "/metrics", &[], ""); let metrics = request(&address, "GET", "/metrics", &[], "");
assert!(metrics.contains("\"attempts\":5"), "{metrics}"); assert!(metrics.contains("\"attempts\":5"), "{metrics}");
@@ -246,6 +261,7 @@ fn failed_durable_commit_rolls_back_visible_state() {
&origin, &origin,
); );
assert!(rejected.starts_with("HTTP/1.1 503"), "{rejected}"); assert!(rejected.starts_with("HTTP/1.1 503"), "{rejected}");
assert!(response_header(&rejected, "x-request-id").starts_with("req-"));
assert!(rejected.contains("{\"code\":\"storage-unavailable\"}")); assert!(rejected.contains("{\"code\":\"storage-unavailable\"}"));
assert!(!rejected.contains("Must Rollback")); assert!(!rejected.contains("Must Rollback"));
assert!(!rejected.contains("demo-csrf")); assert!(!rejected.contains("demo-csrf"));