diff --git a/PLAN.md b/PLAN.md index fa27d94..8e7c112 100644 --- a/PLAN.md +++ b/PLAN.md @@ -60,13 +60,13 @@ encryption, retention, backup, and deployment policy remain host concerns. ## Slice 6 — production integration reference - [ ] **User value:** adopters can copy a proven boundary for durable storage, auth, transactions, security controls, observability, and restart recovery without hemx owning vendor policy. -- **State:** In progress — one public server-first project mutation now proves current bearer authentication, origin/CSRF denial, bounded input, atomic durable commit, rollback on persistence failure, and process-restart recovery. +- **State:** In progress — the public server-first mutation now additionally exposes dependency-aware liveness/readiness, aggregate vendor-neutral metrics, typed secret-free diagnostics, and stale-build fail-closed reload recovery. - **Build:** evolve one existing reference app using ordinary integration adapters; add durable app storage, authenticated/authorized allowed and denied mutations, CSRF/origin checks, transaction rollback, bounded input, structured failures, health/readiness, tracing/metrics hooks, and restart/deploy recovery. - **Refusals:** no built-in database/auth provider, compliance claim, telemetry vendor, deployment system, or repository framework. - **Requirements:** `security/001-009`, `operations/001-008`, `v1_release/003`, existing `adapter/*`, `integration/*`, and `diagnostics/*` contracts. - **Proof:** end-to-end test survives process restart and mixed deployment, proves allowed/denied/rolled-back mutations and redacted diagnostics, and maps each framework-owned ASVS-relevant control to a failing/passing case. -`cargo test -p hemx-saas-example --test production_reference` proves that first production-reference path. Execution cursor: add structured/redacted failure diagnostics plus health/readiness and vendor-neutral tracing/metrics hooks around this mutation, then prove mixed-deploy fail-closed recovery. +`cargo test -p hemx-saas-example --test production_reference` proves current bearer authentication, origin/CSRF denial, bounded input, atomic durable commit, rollback, restart recovery, live-vs-ready dependency failure, redacted structured problem responses, aggregate metrics, and stale-fingerprint rejection followed by current-build recovery. The diagnostic hook accepts only typed outcome and duration fields, preventing request secrets from entering framework-owned diagnostic records. Execution cursor: audit Slice 6 requirement citations against `security/001-009`, `operations/001-008`, `adapter/*`, `integration/*`, and `diagnostics/*`; add only the missing end-to-end control cases, then close Slice 6 if clean. ## Slice 7 — v1 compatibility and closure diff --git a/examples/saas/README.md b/examples/saas/README.md index e55c328..1b4f58c 100644 --- a/examples/saas/README.md +++ b/examples/saas/README.md @@ -8,7 +8,8 @@ What it proves: - auth/session context passed through normal Rust state - CSRF-safe mutation checked before persistence - local atomic-file persistence adapter with rollback and process-restart proof instead of a vendored SQL/auth provider -- a bounded `POST /projects` reference boundary requiring the current bearer session, exact origin, and CSRF token +- a bounded `POST /projects` reference boundary requiring the current bearer session, exact origin, CSRF token, and matching generated build fingerprint when supplied +- `/health/live`, dependency-aware `/health/ready`, and aggregate `/metrics` endpoints with secret-free structured diagnostics - generated form, slot, keyed row, page-swap, and live-status commands - page shell with plain CSS and one explicit metrics island script - compile-time surface generation plus interaction tests diff --git a/examples/saas/src/lib.rs b/examples/saas/src/lib.rs index 7ff2591..5f64953 100644 --- a/examples/saas/src/lib.rs +++ b/examples/saas/src/lib.rs @@ -13,7 +13,9 @@ use std::fs; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::str::FromStr; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; +use std::time::Duration; use ui::dashboard; @@ -172,6 +174,16 @@ impl LocalProjectStore { pub fn list(&self) -> Vec { self.projects.lock().unwrap().clone() } + + fn ready(&self) -> bool { + let Some(path) = self.path.as_deref() else { + return true; + }; + if path.exists() && !path.is_file() { + return false; + } + path.parent().unwrap_or_else(|| Path::new(".")).is_dir() + } } fn persist_projects(path: &Path, projects: &[ProjectRecord]) -> io::Result<()> { @@ -192,10 +204,44 @@ fn persist_projects(path: &Path, projects: &[ProjectRecord]) -> io::Result<()> { Ok(()) } +#[derive(Clone, Copy)] +pub struct MutationDiagnostic { + pub outcome: &'static str, + pub duration_micros: u64, +} + +pub trait DiagnosticSink: Send + Sync { + fn record(&self, diagnostic: MutationDiagnostic); +} + +struct StderrDiagnosticSink; + +impl DiagnosticSink for StderrDiagnosticSink { + fn record(&self, diagnostic: MutationDiagnostic) { + eprintln!( + "event=saas.project_mutation outcome={} duration_micros={}", + diagnostic.outcome, diagnostic.duration_micros + ); + } +} + +#[derive(Default)] +struct MutationMetrics { + attempts: AtomicU64, + succeeded: AtomicU64, + denied: AtomicU64, + invalid: AtomicU64, + mismatch: AtomicU64, + failed: AtomicU64, + duration_micros: AtomicU64, +} + #[derive(Clone)] pub struct AppContext { session: Session, store: LocalProjectStore, + metrics: Arc, + diagnostics: Arc, } impl AppContext { @@ -203,6 +249,8 @@ impl AppContext { Self { session: Session::demo(), store: LocalProjectStore::default(), + metrics: Arc::default(), + diagnostics: Arc::new(StderrDiagnosticSink), } } @@ -212,6 +260,8 @@ impl AppContext { Ok(Self { session, store: LocalProjectStore::durable(path)?, + metrics: Arc::default(), + diagnostics: Arc::new(StderrDiagnosticSink), }) } @@ -241,6 +291,48 @@ impl AppContext { self.store.list() } + pub fn ready(&self) -> bool { + self.store.ready() + } + + pub fn with_diagnostic_sink(mut self, diagnostics: Arc) -> Self { + self.diagnostics = diagnostics; + self + } + + pub fn record_mutation(&self, outcome: &'static str, duration: Duration) { + self.metrics.attempts.fetch_add(1, Ordering::Relaxed); + match outcome { + "succeeded" => &self.metrics.succeeded, + "denied" => &self.metrics.denied, + "invalid" => &self.metrics.invalid, + "mismatch" => &self.metrics.mismatch, + _ => &self.metrics.failed, + } + .fetch_add(1, Ordering::Relaxed); + let duration_micros = duration.as_micros().min(u128::from(u64::MAX)) as u64; + self.metrics + .duration_micros + .fetch_add(duration_micros, Ordering::Relaxed); + self.diagnostics.record(MutationDiagnostic { + outcome, + duration_micros, + }); + } + + pub fn metrics_json(&self) -> String { + format!( + "{{\"project_mutation\":{{\"attempts\":{},\"succeeded\":{},\"denied\":{},\"invalid\":{},\"mismatch\":{},\"failed\":{},\"duration_micros\":{}}}}}", + self.metrics.attempts.load(Ordering::Relaxed), + self.metrics.succeeded.load(Ordering::Relaxed), + self.metrics.denied.load(Ordering::Relaxed), + self.metrics.invalid.load(Ordering::Relaxed), + self.metrics.mismatch.load(Ordering::Relaxed), + self.metrics.failed.load(Ordering::Relaxed), + self.metrics.duration_micros.load(Ordering::Relaxed), + ) + } + pub fn create_project_authorized( &self, name: &str, @@ -477,6 +569,28 @@ mod tests { Selector::parse(value).expect("test selector parses") } + #[derive(Default)] + struct RecordingDiagnostics(Mutex>); + + impl DiagnosticSink for RecordingDiagnostics { + fn record(&self, diagnostic: MutationDiagnostic) { + self.0.lock().unwrap().push(diagnostic); + } + } + + #[test] + fn mutation_diagnostics_are_structured_and_cannot_carry_request_secrets() { + // req: operations/003 req: operations/005 req: security/008 + let diagnostics = Arc::new(RecordingDiagnostics::default()); + let ctx = AppContext::demo().with_diagnostic_sink(diagnostics.clone()); + ctx.record_mutation("denied", Duration::from_micros(7)); + + let recorded = diagnostics.0.lock().unwrap(); + assert_eq!(recorded.len(), 1); + assert_eq!(recorded[0].outcome, "denied"); + assert_eq!(recorded[0].duration_micros, 7); + } + #[test] fn home_page_documents_the_production_app_boundaries() { // req: examples/001 req: auth/001 req: auth/004 req: interop/003 diff --git a/examples/saas/src/main.rs b/examples/saas/src/main.rs index dfa4146..33b8b3c 100644 --- a/examples/saas/src/main.rs +++ b/examples/saas/src/main.rs @@ -11,6 +11,7 @@ use hemx_saas_example::{home_page, live_status, registry, settings_page, ui, App use std::collections::BTreeMap; use std::convert::Infallible; use std::path::PathBuf; +use std::time::Instant; #[tokio::main] async fn main() -> Result<(), Box> { @@ -29,6 +30,9 @@ fn app(ctx: AppContext) -> Router { .route("/", get(home).post(interact)) .route("/settings", get(settings)) .route("/projects", post(create_project)) + .route("/health/live", get(health_live)) + .route("/health/ready", get(health_ready)) + .route("/metrics", get(metrics)) .route("/events", get(events)) .route(runtime_js_path(), get(runtime)) .route("/app.css", get(css)) @@ -75,6 +79,7 @@ async fn create_project( headers: HeaderMap, Form(form): Form>, ) -> Response { + let started = Instant::now(); let bearer = headers .get("authorization") .and_then(|value| value.to_str().ok()) @@ -85,18 +90,90 @@ async fn create_project( .unwrap_or_default(); let name = form.get("name").map(String::as_str).unwrap_or_default(); let csrf = form.get("csrf").map(String::as_str).unwrap_or_default(); - match ctx.create_project_authorized(name, bearer, csrf, origin) { - Ok(_) => (StatusCode::SEE_OTHER, [("location", "/")], "").into_response(), - Err( - error @ (hemx_saas_example::AppError::MissingSession - | hemx_saas_example::AppError::CsrfRejected - | hemx_saas_example::AppError::OriginRejected), - ) => (StatusCode::FORBIDDEN, error.to_string()).into_response(), - Err(error @ hemx_saas_example::AppError::Validation(_)) => { - (StatusCode::BAD_REQUEST, error.to_string()).into_response() + if let Some(client_fingerprint) = headers + .get("x-hemx-fingerprint") + .and_then(|value| value.to_str().ok()) + { + let current_fingerprint = ui::BUILD_FINGERPRINT.0.to_string(); + if client_fingerprint != current_fingerprint { + ctx.record_mutation("mismatch", started.elapsed()); + return Response::builder() + .status(StatusCode::CONFLICT) + .header("content-type", "application/problem+json") + .header("x-hemx-recovery", "reload") + .header("x-hemx-fingerprint", current_fingerprint) + .body(Body::from("{\"code\":\"deployment-mismatch\"}")) + .expect("deployment mismatch response"); } - Err(error) => (StatusCode::SERVICE_UNAVAILABLE, error.to_string()).into_response(), } + let (outcome, response) = match ctx.create_project_authorized(name, bearer, csrf, origin) { + Ok(_) => ( + "succeeded", + (StatusCode::SEE_OTHER, [("location", "/")], "").into_response(), + ), + Err( + hemx_saas_example::AppError::MissingSession + | hemx_saas_example::AppError::CsrfRejected + | hemx_saas_example::AppError::OriginRejected, + ) => ( + "denied", + problem(StatusCode::FORBIDDEN, "authorization-denied"), + ), + Err(hemx_saas_example::AppError::Validation(_)) => ( + "invalid", + problem(StatusCode::BAD_REQUEST, "invalid-project"), + ), + Err(_) => ( + "failed", + problem(StatusCode::SERVICE_UNAVAILABLE, "storage-unavailable"), + ), + }; + ctx.record_mutation(outcome, started.elapsed()); + response +} + +fn problem(status: StatusCode, code: &'static str) -> Response { + Response::builder() + .status(status) + .header("content-type", "application/problem+json") + .body(Body::from(format!("{{\"code\":\"{code}\"}}"))) + .expect("problem response") +} + +// req: operations/007 +async fn health_live() -> Response { + json_response(StatusCode::OK, "{\"status\":\"live\"}".to_owned()) +} + +// req: operations/007 +async fn health_ready(State(ctx): State) -> Response { + if ctx.ready() { + json_response( + StatusCode::OK, + format!( + "{{\"status\":\"ready\",\"fingerprint\":\"{}\"}}", + ui::BUILD_FINGERPRINT.0 + ), + ) + } else { + json_response( + StatusCode::SERVICE_UNAVAILABLE, + "{\"status\":\"not-ready\",\"code\":\"storage-unavailable\"}".to_owned(), + ) + } +} + +// req: operations/005 req: operations/007 +async fn metrics(State(ctx): State) -> Response { + json_response(StatusCode::OK, ctx.metrics_json()) +} + +fn json_response(status: StatusCode, body: String) -> Response { + Response::builder() + .status(status) + .header("content-type", "application/json") + .body(Body::from(body)) + .expect("JSON response") } async fn runtime() -> impl IntoResponse { diff --git a/examples/saas/tests/production_reference.rs b/examples/saas/tests/production_reference.rs index 4cc2aea..2ba15e4 100644 --- a/examples/saas/tests/production_reference.rs +++ b/examples/saas/tests/production_reference.rs @@ -55,19 +55,41 @@ fn request( } fn create(address: &str, name: &str, bearer: &str, csrf: &str, origin: &str) -> String { + create_at_version(address, name, bearer, csrf, origin, None) +} + +fn create_at_version( + address: &str, + name: &str, + bearer: &str, + csrf: &str, + origin: &str, + fingerprint: Option<&str>, +) -> String { + let mut headers = vec![ + ("Authorization", bearer), + ("Origin", origin), + ("Content-Type", "application/x-www-form-urlencoded"), + ]; + if let Some(fingerprint) = fingerprint { + headers.push(("x-hemx-fingerprint", fingerprint)); + } request( address, "POST", "/projects", - &[ - ("Authorization", bearer), - ("Origin", origin), - ("Content-Type", "application/x-www-form-urlencoded"), - ], + &headers, &format!("name={name}&csrf={csrf}"), ) } +fn ready_fingerprint(response: &str) -> &str { + let marker = "\"fingerprint\":\""; + let start = response.find(marker).expect("readiness fingerprint") + marker.len(); + let end = response[start..].find('"').expect("fingerprint end") + start; + &response[start..end] +} + #[test] fn authenticated_project_mutation_is_atomic_and_survives_restart() { // test req: auth/001 req: auth/002 req: auth/004 req: security/004 req: v1_release/003 @@ -77,8 +99,21 @@ fn authenticated_project_mutation_is_atomic_and_survives_restart() { { let _app = start(&address, &store); - for denied in [ - create(&address, "DeniedAuth", "Bearer wrong", "demo-csrf", &origin), + let live = request(&address, "GET", "/health/live", &[], ""); + assert!(live.starts_with("HTTP/1.1 200"), "{live}"); + assert!(live.contains("{\"status\":\"live\"}"), "{live}"); + let ready = request(&address, "GET", "/health/ready", &[], ""); + assert!(ready.starts_with("HTTP/1.1 200"), "{ready}"); + assert!(ready.contains("{\"status\":\"ready\","), "{ready}"); + + let denied_responses = [ + create( + &address, + "DeniedAuth", + "Bearer secret-auth-material", + "demo-csrf", + &origin, + ), create( &address, "DeniedCsrf", @@ -93,8 +128,14 @@ fn authenticated_project_mutation_is_atomic_and_survives_restart() { "demo-csrf", "https://attacker.invalid", ), - ] { + ]; + for denied in &denied_responses { assert!(denied.starts_with("HTTP/1.1 403"), "{denied}"); + assert!(denied.contains("{\"code\":\"authorization-denied\"}")); + assert!(!denied.contains("Denied")); + assert!(!denied.contains("demo-csrf")); + assert!(!denied.contains("secret-auth-material")); + assert!(!denied.contains("attacker.invalid")); } let wrong_content_type = request( &address, @@ -128,16 +169,50 @@ fn authenticated_project_mutation_is_atomic_and_survives_restart() { assert!(!before.contains("DeniedCsrf")); assert!(!before.contains("DeniedOrigin")); assert!(!before.contains("WrongType")); + let denied_metrics = request(&address, "GET", "/metrics", &[], ""); + assert!( + denied_metrics.starts_with("HTTP/1.1 200"), + "{denied_metrics}" + ); + assert!( + denied_metrics.contains("\"attempts\":3"), + "{denied_metrics}" + ); + assert!(denied_metrics.contains("\"denied\":3"), "{denied_metrics}"); + assert!(!denied_metrics.contains("Denied")); + assert!(!denied_metrics.contains("demo-csrf")); + assert!(!denied_metrics.contains("secret-auth-material")); - let allowed = create( + let stale = create_at_version( + &address, + "Stale%20Project", + "Bearer demo-session", + "demo-csrf", + &origin, + Some("0"), + ); + assert!(stale.starts_with("HTTP/1.1 409"), "{stale}"); + assert!(stale.contains("{\"code\":\"deployment-mismatch\"}")); + assert!(stale + .to_ascii_lowercase() + .contains("x-hemx-recovery: reload")); + assert!(!request(&address, "GET", "/", &[], "").contains("Stale Project")); + + let fingerprint = ready_fingerprint(&ready); + let allowed = create_at_version( &address, "Durable%20Project", "Bearer demo-session", "demo-csrf", &origin, + Some(fingerprint), ); assert!(allowed.starts_with("HTTP/1.1 303"), "{allowed}"); assert!(request(&address, "GET", "/", &[], "").contains("Durable Project")); + let metrics = request(&address, "GET", "/metrics", &[], ""); + assert!(metrics.contains("\"attempts\":5"), "{metrics}"); + assert!(metrics.contains("\"succeeded\":1"), "{metrics}"); + assert!(metrics.contains("\"mismatch\":1"), "{metrics}"); } { @@ -158,6 +233,10 @@ fn failed_durable_commit_rolls_back_visible_state() { let store = test_path("rollback"); let _app = start(&address, &store); fs::create_dir(&store).expect("block atomic rename destination"); + let not_ready = request(&address, "GET", "/health/ready", &[], ""); + assert!(not_ready.starts_with("HTTP/1.1 503"), "{not_ready}"); + assert!(not_ready.contains("\"code\":\"storage-unavailable\"")); + assert!(request(&address, "GET", "/health/live", &[], "").starts_with("HTTP/1.1 200")); let rejected = create( &address, @@ -167,8 +246,15 @@ fn failed_durable_commit_rolls_back_visible_state() { &origin, ); assert!(rejected.starts_with("HTTP/1.1 503"), "{rejected}"); + assert!(rejected.contains("{\"code\":\"storage-unavailable\"}")); + assert!(!rejected.contains("Must Rollback")); + assert!(!rejected.contains("demo-csrf")); assert!(!request(&address, "GET", "/", &[], "").contains("Must Rollback")); assert!(!store.with_extension("tmp").exists()); + let metrics = request(&address, "GET", "/metrics", &[], ""); + assert!(metrics.contains("\"attempts\":1"), "{metrics}"); + assert!(metrics.contains("\"failed\":1"), "{metrics}"); + assert!(!metrics.contains("Must Rollback")); let _ = fs::remove_dir(store); }