feat(saas): expose redacted operational signals

req: operations/003

req: operations/005

req: operations/007

req: security/008

req: v1_release/003

PLAN.md

@@ -60,13 +60,13 @@ encryption, retention, backup, and deployment policy remain host concerns.
 ## Slice 6 — production integration reference
 
 - [ ] **User value:** adopters can copy a proven boundary for durable storage, auth, transactions, security controls, observability, and restart recovery without hemx owning vendor policy.
-- **State:** In progress — one public server-first project mutation now proves current bearer authentication, origin/CSRF denial, bounded input, atomic durable commit, rollback on persistence failure, and process-restart recovery.
+- **State:** In progress — the public server-first mutation now additionally exposes dependency-aware liveness/readiness, aggregate vendor-neutral metrics, typed secret-free diagnostics, and stale-build fail-closed reload recovery.
 - **Build:** evolve one existing reference app using ordinary integration adapters; add durable app storage, authenticated/authorized allowed and denied mutations, CSRF/origin checks, transaction rollback, bounded input, structured failures, health/readiness, tracing/metrics hooks, and restart/deploy recovery.
 - **Refusals:** no built-in database/auth provider, compliance claim, telemetry vendor, deployment system, or repository framework.
 - **Requirements:** `security/001-009`, `operations/001-008`, `v1_release/003`, existing `adapter/*`, `integration/*`, and `diagnostics/*` contracts.
 - **Proof:** end-to-end test survives process restart and mixed deployment, proves allowed/denied/rolled-back mutations and redacted diagnostics, and maps each framework-owned ASVS-relevant control to a failing/passing case.
 
-`cargo test -p hemx-saas-example --test production_reference` proves that first production-reference path. Execution cursor: add structured/redacted failure diagnostics plus health/readiness and vendor-neutral tracing/metrics hooks around this mutation, then prove mixed-deploy fail-closed recovery.
+`cargo test -p hemx-saas-example --test production_reference` proves current bearer authentication, origin/CSRF denial, bounded input, atomic durable commit, rollback, restart recovery, live-vs-ready dependency failure, redacted structured problem responses, aggregate metrics, and stale-fingerprint rejection followed by current-build recovery. The diagnostic hook accepts only typed outcome and duration fields, preventing request secrets from entering framework-owned diagnostic records. Execution cursor: audit Slice 6 requirement citations against `security/001-009`, `operations/001-008`, `adapter/*`, `integration/*`, and `diagnostics/*`; add only the missing end-to-end control cases, then close Slice 6 if clean.
 
 ## Slice 7 — v1 compatibility and closure
 

examples/saas/README.md

@@ -8,7 +8,8 @@ What it proves:
 - auth/session context passed through normal Rust state
 - CSRF-safe mutation checked before persistence
 - local atomic-file persistence adapter with rollback and process-restart proof instead of a vendored SQL/auth provider
-- a bounded `POST /projects` reference boundary requiring the current bearer session, exact origin, and CSRF token
+- a bounded `POST /projects` reference boundary requiring the current bearer session, exact origin, CSRF token, and matching generated build fingerprint when supplied
+- `/health/live`, dependency-aware `/health/ready`, and aggregate `/metrics` endpoints with secret-free structured diagnostics
 - generated form, slot, keyed row, page-swap, and live-status commands
 - page shell with plain CSS and one explicit metrics island script
 - compile-time surface generation plus interaction tests

examples/saas/src/lib.rs

@@ -13,7 +13,9 @@ use std::fs;
 use std::io::{self, Write};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use ui::dashboard;
 
@@ -172,6 +174,16 @@ impl LocalProjectStore {
     pub fn list(&self) -> Vec<ProjectRecord> {
         self.projects.lock().unwrap().clone()
     }
+
+    fn ready(&self) -> bool {
+        let Some(path) = self.path.as_deref() else {
+            return true;
+        };
+        if path.exists() && !path.is_file() {
+            return false;
+        }
+        path.parent().unwrap_or_else(|| Path::new(".")).is_dir()
+    }
 }
 
 fn persist_projects(path: &Path, projects: &[ProjectRecord]) -> io::Result<()> {
@@ -192,10 +204,44 @@ fn persist_projects(path: &Path, projects: &[ProjectRecord]) -> io::Result<()> {
     Ok(())
 }
 
+#[derive(Clone, Copy)]
+pub struct MutationDiagnostic {
+    pub outcome: &'static str,
+    pub duration_micros: u64,
+}
+
+pub trait DiagnosticSink: Send + Sync {
+    fn record(&self, diagnostic: MutationDiagnostic);
+}
+
+struct StderrDiagnosticSink;
+
+impl DiagnosticSink for StderrDiagnosticSink {
+    fn record(&self, diagnostic: MutationDiagnostic) {
+        eprintln!(
+            "event=saas.project_mutation outcome={} duration_micros={}",
+            diagnostic.outcome, diagnostic.duration_micros
+        );
+    }
+}
+
+#[derive(Default)]
+struct MutationMetrics {
+    attempts: AtomicU64,
+    succeeded: AtomicU64,
+    denied: AtomicU64,
+    invalid: AtomicU64,
+    mismatch: AtomicU64,
+    failed: AtomicU64,
+    duration_micros: AtomicU64,
+}
+
 #[derive(Clone)]
 pub struct AppContext {
     session: Session,
     store: LocalProjectStore,
+    metrics: Arc<MutationMetrics>,
+    diagnostics: Arc<dyn DiagnosticSink>,
 }
 
 impl AppContext {
@@ -203,6 +249,8 @@ impl AppContext {
         Self {
             session: Session::demo(),
             store: LocalProjectStore::default(),
+            metrics: Arc::default(),
+            diagnostics: Arc::new(StderrDiagnosticSink),
         }
     }
 
@@ -212,6 +260,8 @@ impl AppContext {
         Ok(Self {
             session,
             store: LocalProjectStore::durable(path)?,
+            metrics: Arc::default(),
+            diagnostics: Arc::new(StderrDiagnosticSink),
         })
     }
 
@@ -241,6 +291,48 @@ impl AppContext {
         self.store.list()
     }
 
+    pub fn ready(&self) -> bool {
+        self.store.ready()
+    }
+
+    pub fn with_diagnostic_sink(mut self, diagnostics: Arc<dyn DiagnosticSink>) -> Self {
+        self.diagnostics = diagnostics;
+        self
+    }
+
+    pub fn record_mutation(&self, outcome: &'static str, duration: Duration) {
+        self.metrics.attempts.fetch_add(1, Ordering::Relaxed);
+        match outcome {
+            "succeeded" => &self.metrics.succeeded,
+            "denied" => &self.metrics.denied,
+            "invalid" => &self.metrics.invalid,
+            "mismatch" => &self.metrics.mismatch,
+            _ => &self.metrics.failed,
+        }
+        .fetch_add(1, Ordering::Relaxed);
+        let duration_micros = duration.as_micros().min(u128::from(u64::MAX)) as u64;
+        self.metrics
+            .duration_micros
+            .fetch_add(duration_micros, Ordering::Relaxed);
+        self.diagnostics.record(MutationDiagnostic {
+            outcome,
+            duration_micros,
+        });
+    }
+
+    pub fn metrics_json(&self) -> String {
+        format!(
+            "{{\"project_mutation\":{{\"attempts\":{},\"succeeded\":{},\"denied\":{},\"invalid\":{},\"mismatch\":{},\"failed\":{},\"duration_micros\":{}}}}}",
+            self.metrics.attempts.load(Ordering::Relaxed),
+            self.metrics.succeeded.load(Ordering::Relaxed),
+            self.metrics.denied.load(Ordering::Relaxed),
+            self.metrics.invalid.load(Ordering::Relaxed),
+            self.metrics.mismatch.load(Ordering::Relaxed),
+            self.metrics.failed.load(Ordering::Relaxed),
+            self.metrics.duration_micros.load(Ordering::Relaxed),
+        )
+    }
+
     pub fn create_project_authorized(
         &self,
         name: &str,
@@ -477,6 +569,28 @@ mod tests {
         Selector::parse(value).expect("test selector parses")
     }
 
+    #[derive(Default)]
+    struct RecordingDiagnostics(Mutex<Vec<MutationDiagnostic>>);
+
+    impl DiagnosticSink for RecordingDiagnostics {
+        fn record(&self, diagnostic: MutationDiagnostic) {
+            self.0.lock().unwrap().push(diagnostic);
+        }
+    }
+
+    #[test]
+    fn mutation_diagnostics_are_structured_and_cannot_carry_request_secrets() {
+        // req: operations/003 req: operations/005 req: security/008
+        let diagnostics = Arc::new(RecordingDiagnostics::default());
+        let ctx = AppContext::demo().with_diagnostic_sink(diagnostics.clone());
+        ctx.record_mutation("denied", Duration::from_micros(7));
+
+        let recorded = diagnostics.0.lock().unwrap();
+        assert_eq!(recorded.len(), 1);
+        assert_eq!(recorded[0].outcome, "denied");
+        assert_eq!(recorded[0].duration_micros, 7);
+    }
+
     #[test]
     fn home_page_documents_the_production_app_boundaries() {
         // req: examples/001 req: auth/001 req: auth/004 req: interop/003

examples/saas/src/main.rs

@@ -11,6 +11,7 @@ use hemx_saas_example::{home_page, live_status, registry, settings_page, ui, App
 use std::collections::BTreeMap;
 use std::convert::Infallible;
 use std::path::PathBuf;
+use std::time::Instant;
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -29,6 +30,9 @@ fn app(ctx: AppContext) -> Router {
         .route("/", get(home).post(interact))
         .route("/settings", get(settings))
         .route("/projects", post(create_project))
+        .route("/health/live", get(health_live))
+        .route("/health/ready", get(health_ready))
+        .route("/metrics", get(metrics))
         .route("/events", get(events))
         .route(runtime_js_path(), get(runtime))
         .route("/app.css", get(css))
@@ -75,6 +79,7 @@ async fn create_project(
     headers: HeaderMap,
     Form(form): Form<BTreeMap<String, String>>,
 ) -> Response {
+    let started = Instant::now();
     let bearer = headers
         .get("authorization")
         .and_then(|value| value.to_str().ok())
@@ -85,18 +90,90 @@ async fn create_project(
         .unwrap_or_default();
     let name = form.get("name").map(String::as_str).unwrap_or_default();
     let csrf = form.get("csrf").map(String::as_str).unwrap_or_default();
-    match ctx.create_project_authorized(name, bearer, csrf, origin) {
+    if let Some(client_fingerprint) = headers
-        Ok(_) => (StatusCode::SEE_OTHER, [("location", "/")], "").into_response(),
+        .get("x-hemx-fingerprint")
-        Err(
+        .and_then(|value| value.to_str().ok())
-            error @ (hemx_saas_example::AppError::MissingSession
+    {
-            | hemx_saas_example::AppError::CsrfRejected
+        let current_fingerprint = ui::BUILD_FINGERPRINT.0.to_string();
-            | hemx_saas_example::AppError::OriginRejected),
+        if client_fingerprint != current_fingerprint {
-        ) => (StatusCode::FORBIDDEN, error.to_string()).into_response(),
+            ctx.record_mutation("mismatch", started.elapsed());
-        Err(error @ hemx_saas_example::AppError::Validation(_)) => {
+            return Response::builder()
-            (StatusCode::BAD_REQUEST, error.to_string()).into_response()
+                .status(StatusCode::CONFLICT)
+                .header("content-type", "application/problem+json")
+                .header("x-hemx-recovery", "reload")
+                .header("x-hemx-fingerprint", current_fingerprint)
+                .body(Body::from("{\"code\":\"deployment-mismatch\"}"))
+                .expect("deployment mismatch response");
         }
-        Err(error) => (StatusCode::SERVICE_UNAVAILABLE, error.to_string()).into_response(),
     }
+    let (outcome, response) = match ctx.create_project_authorized(name, bearer, csrf, origin) {
+        Ok(_) => (
+            "succeeded",
+            (StatusCode::SEE_OTHER, [("location", "/")], "").into_response(),
+        ),
+        Err(
+            hemx_saas_example::AppError::MissingSession
+            | hemx_saas_example::AppError::CsrfRejected
+            | hemx_saas_example::AppError::OriginRejected,
+        ) => (
+            "denied",
+            problem(StatusCode::FORBIDDEN, "authorization-denied"),
+        ),
+        Err(hemx_saas_example::AppError::Validation(_)) => (
+            "invalid",
+            problem(StatusCode::BAD_REQUEST, "invalid-project"),
+        ),
+        Err(_) => (
+            "failed",
+            problem(StatusCode::SERVICE_UNAVAILABLE, "storage-unavailable"),
+        ),
+    };
+    ctx.record_mutation(outcome, started.elapsed());
+    response
+}
+
+fn problem(status: StatusCode, code: &'static str) -> Response {
+    Response::builder()
+        .status(status)
+        .header("content-type", "application/problem+json")
+        .body(Body::from(format!("{{\"code\":\"{code}\"}}")))
+        .expect("problem response")
+}
+
+// req: operations/007
+async fn health_live() -> Response {
+    json_response(StatusCode::OK, "{\"status\":\"live\"}".to_owned())
+}
+
+// req: operations/007
+async fn health_ready(State(ctx): State<AppContext>) -> Response {
+    if ctx.ready() {
+        json_response(
+            StatusCode::OK,
+            format!(
+                "{{\"status\":\"ready\",\"fingerprint\":\"{}\"}}",
+                ui::BUILD_FINGERPRINT.0
+            ),
+        )
+    } else {
+        json_response(
+            StatusCode::SERVICE_UNAVAILABLE,
+            "{\"status\":\"not-ready\",\"code\":\"storage-unavailable\"}".to_owned(),
+        )
+    }
+}
+
+// req: operations/005 req: operations/007
+async fn metrics(State(ctx): State<AppContext>) -> Response {
+    json_response(StatusCode::OK, ctx.metrics_json())
+}
+
+fn json_response(status: StatusCode, body: String) -> Response {
+    Response::builder()
+        .status(status)
+        .header("content-type", "application/json")
+        .body(Body::from(body))
+        .expect("JSON response")
 }
 
 async fn runtime() -> impl IntoResponse {

examples/saas/tests/production_reference.rs

@@ -55,19 +55,41 @@ fn request(
 }
 
 fn create(address: &str, name: &str, bearer: &str, csrf: &str, origin: &str) -> String {
+    create_at_version(address, name, bearer, csrf, origin, None)
+}
+
+fn create_at_version(
+    address: &str,
+    name: &str,
+    bearer: &str,
+    csrf: &str,
+    origin: &str,
+    fingerprint: Option<&str>,
+) -> String {
+    let mut headers = vec![
+        ("Authorization", bearer),
+        ("Origin", origin),
+        ("Content-Type", "application/x-www-form-urlencoded"),
+    ];
+    if let Some(fingerprint) = fingerprint {
+        headers.push(("x-hemx-fingerprint", fingerprint));
+    }
     request(
         address,
         "POST",
         "/projects",
-        &[
+        &headers,
-            ("Authorization", bearer),
-            ("Origin", origin),
-            ("Content-Type", "application/x-www-form-urlencoded"),
-        ],
         &format!("name={name}&csrf={csrf}"),
     )
 }
 
+fn ready_fingerprint(response: &str) -> &str {
+    let marker = "\"fingerprint\":\"";
+    let start = response.find(marker).expect("readiness fingerprint") + marker.len();
+    let end = response[start..].find('"').expect("fingerprint end") + start;
+    &response[start..end]
+}
+
 #[test]
 fn authenticated_project_mutation_is_atomic_and_survives_restart() {
     // test req: auth/001 req: auth/002 req: auth/004 req: security/004 req: v1_release/003
@@ -77,8 +99,21 @@ fn authenticated_project_mutation_is_atomic_and_survives_restart() {
 
     {
         let _app = start(&address, &store);
-        for denied in [
+        let live = request(&address, "GET", "/health/live", &[], "");
-            create(&address, "DeniedAuth", "Bearer wrong", "demo-csrf", &origin),
+        assert!(live.starts_with("HTTP/1.1 200"), "{live}");
+        assert!(live.contains("{\"status\":\"live\"}"), "{live}");
+        let ready = request(&address, "GET", "/health/ready", &[], "");
+        assert!(ready.starts_with("HTTP/1.1 200"), "{ready}");
+        assert!(ready.contains("{\"status\":\"ready\","), "{ready}");
+
+        let denied_responses = [
+            create(
+                &address,
+                "DeniedAuth",
+                "Bearer secret-auth-material",
+                "demo-csrf",
+                &origin,
+            ),
             create(
                 &address,
                 "DeniedCsrf",
@@ -93,8 +128,14 @@ fn authenticated_project_mutation_is_atomic_and_survives_restart() {
                 "demo-csrf",
                 "https://attacker.invalid",
             ),
-        ] {
+        ];
+        for denied in &denied_responses {
             assert!(denied.starts_with("HTTP/1.1 403"), "{denied}");
+            assert!(denied.contains("{\"code\":\"authorization-denied\"}"));
+            assert!(!denied.contains("Denied"));
+            assert!(!denied.contains("demo-csrf"));
+            assert!(!denied.contains("secret-auth-material"));
+            assert!(!denied.contains("attacker.invalid"));
         }
         let wrong_content_type = request(
             &address,
@@ -128,16 +169,50 @@ fn authenticated_project_mutation_is_atomic_and_survives_restart() {
         assert!(!before.contains("DeniedCsrf"));
         assert!(!before.contains("DeniedOrigin"));
         assert!(!before.contains("WrongType"));
+        let denied_metrics = request(&address, "GET", "/metrics", &[], "");
+        assert!(
+            denied_metrics.starts_with("HTTP/1.1 200"),
+            "{denied_metrics}"
+        );
+        assert!(
+            denied_metrics.contains("\"attempts\":3"),
+            "{denied_metrics}"
+        );
+        assert!(denied_metrics.contains("\"denied\":3"), "{denied_metrics}");
+        assert!(!denied_metrics.contains("Denied"));
+        assert!(!denied_metrics.contains("demo-csrf"));
+        assert!(!denied_metrics.contains("secret-auth-material"));
 
-        let allowed = create(
+        let stale = create_at_version(
+            &address,
+            "Stale%20Project",
+            "Bearer demo-session",
+            "demo-csrf",
+            &origin,
+            Some("0"),
+        );
+        assert!(stale.starts_with("HTTP/1.1 409"), "{stale}");
+        assert!(stale.contains("{\"code\":\"deployment-mismatch\"}"));
+        assert!(stale
+            .to_ascii_lowercase()
+            .contains("x-hemx-recovery: reload"));
+        assert!(!request(&address, "GET", "/", &[], "").contains("Stale Project"));
+
+        let fingerprint = ready_fingerprint(&ready);
+        let allowed = create_at_version(
             &address,
             "Durable%20Project",
             "Bearer demo-session",
             "demo-csrf",
             &origin,
+            Some(fingerprint),
         );
         assert!(allowed.starts_with("HTTP/1.1 303"), "{allowed}");
         assert!(request(&address, "GET", "/", &[], "").contains("Durable Project"));
+        let metrics = request(&address, "GET", "/metrics", &[], "");
+        assert!(metrics.contains("\"attempts\":5"), "{metrics}");
+        assert!(metrics.contains("\"succeeded\":1"), "{metrics}");
+        assert!(metrics.contains("\"mismatch\":1"), "{metrics}");
     }
 
     {
@@ -158,6 +233,10 @@ fn failed_durable_commit_rolls_back_visible_state() {
     let store = test_path("rollback");
     let _app = start(&address, &store);
     fs::create_dir(&store).expect("block atomic rename destination");
+    let not_ready = request(&address, "GET", "/health/ready", &[], "");
+    assert!(not_ready.starts_with("HTTP/1.1 503"), "{not_ready}");
+    assert!(not_ready.contains("\"code\":\"storage-unavailable\""));
+    assert!(request(&address, "GET", "/health/live", &[], "").starts_with("HTTP/1.1 200"));
 
     let rejected = create(
         &address,
@@ -167,8 +246,15 @@ fn failed_durable_commit_rolls_back_visible_state() {
         &origin,
     );
     assert!(rejected.starts_with("HTTP/1.1 503"), "{rejected}");
+    assert!(rejected.contains("{\"code\":\"storage-unavailable\"}"));
+    assert!(!rejected.contains("Must Rollback"));
+    assert!(!rejected.contains("demo-csrf"));
     assert!(!request(&address, "GET", "/", &[], "").contains("Must Rollback"));
     assert!(!store.with_extension("tmp").exists());
+    let metrics = request(&address, "GET", "/metrics", &[], "");
+    assert!(metrics.contains("\"attempts\":1"), "{metrics}");
+    assert!(metrics.contains("\"failed\":1"), "{metrics}");
+    assert!(!metrics.contains("Must Rollback"));
 
     let _ = fs::remove_dir(store);
 }