From f31885ba07b305ff52cda66c74dc763c30c99481 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 15 Sep 2026 15:04:48 -0600 Subject: [PATCH] test: fixed tests for the new finding-verifier agent changes --- src/config/request_context.rs | 68 +++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/src/config/request_context.rs b/src/config/request_context.rs index af42090..16fca77 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -14601,6 +14601,16 @@ mod tests { .map(str::to_owned) } + fn fv_soft_note(out: &serde_json::Value) -> Option { + out["verdict_out"]["attention"] + .as_array() + .unwrap() + .iter() + .filter_map(|a| a.as_str()) + .find(|a| a.starts_with("Verifier: ") && a.contains("could not be verified")) + .map(str::to_owned) + } + #[test] fn code_reviewer_verdict_fv_duplicate_unknown_faults_count_individually() { if !cmd_available("python3") { @@ -14609,6 +14619,9 @@ mod tests { } // finding-verifier's crash guard emits every fault entry under the // same "unknown" id; an id-indexed dict would collapse them to one. + // Non-sentinel faults are SOFT: the findings stand unverified with a + // human-attention note, but the lane is NOT flagged as a re-runnable + // pipeline fault. let entry = json!({ "id": "unknown", "verdict": "UNVERIFIABLE", @@ -14622,10 +14635,23 @@ mod tests { }); let out = run_code_reviewer_script("verdict.py", &state); assert_eq!(out["verdict_out"]["verdict"], "NEEDS-HUMAN", "{out}"); - let fault = fv_degraded_fault(&out).unwrap_or_else(|| panic!("{out}")); assert!( - fault.contains("2 verifier verdict(s)"), - "each crash-guard entry must count: {fault}" + fv_degraded_fault(&out).is_none(), + "crash-guard entries are soft faults, not a degraded lane: {out}" + ); + let note = fv_soft_note(&out).unwrap_or_else(|| panic!("{out}")); + assert!( + note.contains("2 finding(s) could not be verified"), + "each crash-guard entry must count: {note}" + ); + let reason = out["verdict_out"]["reason"].as_str().unwrap(); + assert!( + reason.contains("2 finding(s) unverified (verifier branch fault)"), + "the reason must count the unverified findings: {out}" + ); + assert!( + !reason.contains("pipeline fault(s) recorded"), + "soft faults must not trip the gauntlet's degraded-run anchor: {out}" ); } @@ -14638,6 +14664,8 @@ mod tests { // finding-verifier completes "successfully" with parse_fault's // sentinel entry inside the payload — not an "Agent node failed:" // banner — so the gate must read the sentinel id as a fault. + // The sentinel means the verifier verified NOTHING: this is the HARD, + // lane-degrading path. let state = json!({ "changed_files": ["a.rs"], "domain_reports": ["clean report. DOMAIN_REVIEW_COMPLETE"], @@ -14665,8 +14693,12 @@ mod tests { panic!("the degraded-verification fault must be in attention: {out}") }); assert!( - fault.contains("1 verifier verdict(s) carry a pipeline fault"), - "the fault must count the faulted verdicts: {fault}" + fault.contains("the verifier verified nothing"), + "the fault must state that nothing was verified: {fault}" + ); + assert!( + fault.contains("parse failed"), + "the fault must carry the sentinel's detail: {fault}" ); } @@ -14678,6 +14710,10 @@ mod tests { } // verify_fault's per-finding entry keeps the real finding id; the // fault is recognized by the note's PIPELINE-FAULT prefix. + // Per-finding faults are SOFT: the finding stands unverified with a + // human-attention note instead of a lane-degrading pipeline fault + // (re-running an entire ~1h lane over a few unverified findings is + // the wrong trade). let state = json!({ "changed_files": ["a.rs"], "domain_reports": ["clean report. DOMAIN_REVIEW_COMPLETE"], @@ -14694,12 +14730,24 @@ mod tests { v["verdict"], "NEEDS-HUMAN", "a lane-faulted verifier verdict must block: {out}" ); - let fault = fv_degraded_fault(&out).unwrap_or_else(|| { - panic!("the degraded-verification fault must be in attention: {out}") - }); assert!( - fault.contains("verifier lane failed after retries"), - "the fault must carry the verifier's note: {fault}" + fv_degraded_fault(&out).is_none(), + "a per-finding fault must not degrade the lane: {out}" + ); + let note = fv_soft_note(&out) + .unwrap_or_else(|| panic!("the unverified-findings note must be in attention: {out}")); + assert!( + note.contains("verifier lane failed after retries"), + "the note must carry the verifier's fault detail: {note}" + ); + let reason = v["reason"].as_str().unwrap(); + assert!( + reason.contains("1 finding(s) unverified (verifier branch fault)"), + "the reason must count the unverified finding: {out}" + ); + assert!( + !reason.contains("pipeline fault(s) recorded"), + "soft faults must not trip the gauntlet's degraded-run anchor: {out}" ); }