feat(saas): expose redacted operational signals

req: operations/003

req: operations/005

req: operations/007

req: security/008

req: v1_release/003

examples/saas/tests/production_reference.rs

@@ -55,19 +55,41 @@ fn request(
 }
 
 fn create(address: &str, name: &str, bearer: &str, csrf: &str, origin: &str) -> String {
+    create_at_version(address, name, bearer, csrf, origin, None)
+}
+
+fn create_at_version(
+    address: &str,
+    name: &str,
+    bearer: &str,
+    csrf: &str,
+    origin: &str,
+    fingerprint: Option<&str>,
+) -> String {
+    let mut headers = vec![
+        ("Authorization", bearer),
+        ("Origin", origin),
+        ("Content-Type", "application/x-www-form-urlencoded"),
+    ];
+    if let Some(fingerprint) = fingerprint {
+        headers.push(("x-hemx-fingerprint", fingerprint));
+    }
     request(
         address,
         "POST",
         "/projects",
-        &[
-            ("Authorization", bearer),
-            ("Origin", origin),
-            ("Content-Type", "application/x-www-form-urlencoded"),
-        ],
+        &headers,
         &format!("name={name}&csrf={csrf}"),
     )
 }
 
+fn ready_fingerprint(response: &str) -> &str {
+    let marker = "\"fingerprint\":\"";
+    let start = response.find(marker).expect("readiness fingerprint") + marker.len();
+    let end = response[start..].find('"').expect("fingerprint end") + start;
+    &response[start..end]
+}
+
 #[test]
 fn authenticated_project_mutation_is_atomic_and_survives_restart() {
     // test req: auth/001 req: auth/002 req: auth/004 req: security/004 req: v1_release/003
@@ -77,8 +99,21 @@ fn authenticated_project_mutation_is_atomic_and_survives_restart() {
 
     {
         let _app = start(&address, &store);
-        for denied in [
-            create(&address, "DeniedAuth", "Bearer wrong", "demo-csrf", &origin),
+        let live = request(&address, "GET", "/health/live", &[], "");
+        assert!(live.starts_with("HTTP/1.1 200"), "{live}");
+        assert!(live.contains("{\"status\":\"live\"}"), "{live}");
+        let ready = request(&address, "GET", "/health/ready", &[], "");
+        assert!(ready.starts_with("HTTP/1.1 200"), "{ready}");
+        assert!(ready.contains("{\"status\":\"ready\","), "{ready}");
+
+        let denied_responses = [
+            create(
+                &address,
+                "DeniedAuth",
+                "Bearer secret-auth-material",
+                "demo-csrf",
+                &origin,
+            ),
             create(
                 &address,
                 "DeniedCsrf",
@@ -93,8 +128,14 @@ fn authenticated_project_mutation_is_atomic_and_survives_restart() {
                 "demo-csrf",
                 "https://attacker.invalid",
             ),
-        ] {
+        ];
+        for denied in &denied_responses {
             assert!(denied.starts_with("HTTP/1.1 403"), "{denied}");
+            assert!(denied.contains("{\"code\":\"authorization-denied\"}"));
+            assert!(!denied.contains("Denied"));
+            assert!(!denied.contains("demo-csrf"));
+            assert!(!denied.contains("secret-auth-material"));
+            assert!(!denied.contains("attacker.invalid"));
         }
         let wrong_content_type = request(
             &address,
@@ -128,16 +169,50 @@ fn authenticated_project_mutation_is_atomic_and_survives_restart() {
         assert!(!before.contains("DeniedCsrf"));
         assert!(!before.contains("DeniedOrigin"));
         assert!(!before.contains("WrongType"));
+        let denied_metrics = request(&address, "GET", "/metrics", &[], "");
+        assert!(
+            denied_metrics.starts_with("HTTP/1.1 200"),
+            "{denied_metrics}"
+        );
+        assert!(
+            denied_metrics.contains("\"attempts\":3"),
+            "{denied_metrics}"
+        );
+        assert!(denied_metrics.contains("\"denied\":3"), "{denied_metrics}");
+        assert!(!denied_metrics.contains("Denied"));
+        assert!(!denied_metrics.contains("demo-csrf"));
+        assert!(!denied_metrics.contains("secret-auth-material"));
 
-        let allowed = create(
+        let stale = create_at_version(
+            &address,
+            "Stale%20Project",
+            "Bearer demo-session",
+            "demo-csrf",
+            &origin,
+            Some("0"),
+        );
+        assert!(stale.starts_with("HTTP/1.1 409"), "{stale}");
+        assert!(stale.contains("{\"code\":\"deployment-mismatch\"}"));
+        assert!(stale
+            .to_ascii_lowercase()
+            .contains("x-hemx-recovery: reload"));
+        assert!(!request(&address, "GET", "/", &[], "").contains("Stale Project"));
+
+        let fingerprint = ready_fingerprint(&ready);
+        let allowed = create_at_version(
             &address,
             "Durable%20Project",
             "Bearer demo-session",
             "demo-csrf",
             &origin,
+            Some(fingerprint),
         );
         assert!(allowed.starts_with("HTTP/1.1 303"), "{allowed}");
         assert!(request(&address, "GET", "/", &[], "").contains("Durable Project"));
+        let metrics = request(&address, "GET", "/metrics", &[], "");
+        assert!(metrics.contains("\"attempts\":5"), "{metrics}");
+        assert!(metrics.contains("\"succeeded\":1"), "{metrics}");
+        assert!(metrics.contains("\"mismatch\":1"), "{metrics}");
     }
 
     {
@@ -158,6 +233,10 @@ fn failed_durable_commit_rolls_back_visible_state() {
     let store = test_path("rollback");
     let _app = start(&address, &store);
     fs::create_dir(&store).expect("block atomic rename destination");
+    let not_ready = request(&address, "GET", "/health/ready", &[], "");
+    assert!(not_ready.starts_with("HTTP/1.1 503"), "{not_ready}");
+    assert!(not_ready.contains("\"code\":\"storage-unavailable\""));
+    assert!(request(&address, "GET", "/health/live", &[], "").starts_with("HTTP/1.1 200"));
 
     let rejected = create(
         &address,
@@ -167,8 +246,15 @@ fn failed_durable_commit_rolls_back_visible_state() {
         &origin,
     );
     assert!(rejected.starts_with("HTTP/1.1 503"), "{rejected}");
+    assert!(rejected.contains("{\"code\":\"storage-unavailable\"}"));
+    assert!(!rejected.contains("Must Rollback"));
+    assert!(!rejected.contains("demo-csrf"));
     assert!(!request(&address, "GET", "/", &[], "").contains("Must Rollback"));
     assert!(!store.with_extension("tmp").exists());
+    let metrics = request(&address, "GET", "/metrics", &[], "");
+    assert!(metrics.contains("\"attempts\":1"), "{metrics}");
+    assert!(metrics.contains("\"failed\":1"), "{metrics}");
+    assert!(!metrics.contains("Must Rollback"));
 
     let _ = fs::remove_dir(store);
 }