From 02672547ee2fc8e15ebe570b5c1d7ea18f97cafd Mon Sep 17 00:00:00 2001
From: Bendik Aagaard Lynghaug <bendik.lynghaug@gmail.com>
Date: Mon, 21 Sep 2026 19:16:28 +0200
Subject: [PATCH] simulate: play every case through every state, judge only
 what a model can

`portal-trainer simulate` reads the model `question_lint --needs-sim`
exports. A visitor model picks an alternative and fills the form in a
manner that varies by run; the case is then walked along the handling
group's own desk buttons to a finished state, a different one each
run, so every state and button is exercised. At each button the
handler model is asked one thing: with only what the form collected,
could this role really do what the button claims? Its question goes
back to the visitor, who may not know.

Who wins is deliberately not simulated. The first version let the
handler decide: qwen3:8b sent every case to "won", and after one
sentence of prompt change, every case to "lost". Outcomes belong to
the live bucket; `success` in needs.yaml is what a report over real
records counts.

`compare` sets two score files side by side. engines/ollama.sh honours
SEED, FORMAT=json and TEMPERATURE so the same case replays the same
way against two versions of a site. generator/reference.md documents
`success` and `stages`.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L4jrCgLiKKHAEFuZUJjckH
---
 engines/ollama.sh      |   9 +-
 generator/reference.md |   6 +
 src/main.rs            |  14 ++
 src/simulate.rs        | 554 +++++++++++++++++++++++++++++++++++++++++
 4 files changed, 581 insertions(+), 2 deletions(-)
 create mode 100644 src/simulate.rs

diff --git a/engines/ollama.sh b/engines/ollama.sh
index 707d60c..75bc0de 100755
--- a/engines/ollama.sh
+++ b/engines/ollama.sh
@@ -2,11 +2,16 @@
 # LLM backend for portal-trainer on a local Ollama: prompt on stdin,
 # text on stdout. MODEL, OLLAMA_HOST, NUM_CTX and TIMEOUT override.
 # Thinking is off: the generator wants files, the answerer wants JSONL.
+# The simulator sets SEED (so the same case replays the same way
+# against two versions of a site), FORMAT=json and TEMPERATURE.
 set -eu
 jq -Rs --arg model "${MODEL:-qwen3:8b}" --argjson ctx "${NUM_CTX:-32768}" \
+  --argjson temp "${TEMPERATURE:-0.2}" --arg seed "${SEED:-}" --arg format "${FORMAT:-}" \
   '{model: $model, stream: false, think: false,
-    options: {num_ctx: $ctx, temperature: 0.2},
-    messages: [{role: "user", content: .}]}' |
+    options: ({num_ctx: $ctx, temperature: $temp}
+              + (if $seed == "" then {} else {seed: ($seed | tonumber)} end)),
+    messages: [{role: "user", content: .}]}
+   + (if $format == "json" then {format: "json"} else {} end)' |
 curl -sS --fail-with-body --max-time "${TIMEOUT:-1800}" \
   "${OLLAMA_HOST:-http://127.0.0.1:11434}/api/chat" \
   -H "content-type: application/json" --data-binary @- |
diff --git a/generator/reference.md b/generator/reference.md
index 67554ff..b5afaf1 100644
--- a/generator/reference.md
+++ b/generator/reference.md
@@ -36,6 +36,12 @@ needs:
     done_when: [<state>, <state>]   # every state that counts as finished, good or bad
     max_steps: 1           # submissions before the record exists (1 or 2)
     max_fields: 4          # required fields the visitor will tolerate across the path
+    success: [<state>]     # the done_when states that are a good outcome for the business
+    stages: [open, <stage>, <stage>, <end>, <end>]
+                           # EVERY stage the business said a case goes through, first to last,
+                           # read off their own sentences one verb at a time ("they come in, we
+                           # go and look, we send a quote, we win or lose it" is five stages).
+                           # The state machine must have a state for each, or the verifier fails.
 personas:
   - id: kebab-case
     need: <need id>
diff --git a/src/main.rs b/src/main.rs
index 3341eb9..761f3b0 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -22,6 +22,8 @@ use std::process::{Command, Stdio};
 
 use anyhow::{anyhow, bail, Context};
 
+mod simulate;
+
 const REFERENCE: &str = include_str!("../generator/reference.md");
 
 struct Opts {
@@ -38,6 +40,11 @@ fn main() -> anyhow::Result<()> {
     // for grading an existing site with any backend. Answers go to
     // stdout, ready for `question_lint --needs-score`.
     let argv: Vec<String> = std::env::args().skip(1).collect();
+    match argv.first().map(String::as_str) {
+        Some("simulate") => return simulate::run(&argv),
+        Some("compare") => return simulate::compare(&argv),
+        _ => {}
+    }
     if argv.first().map(String::as_str) == Some("answer") {
         let value = |flag: &str| {
             argv.iter()
@@ -209,9 +216,16 @@ fn parse_args() -> anyhow::Result<Opts> {
 
 /// Prompt on stdin, text on stdout.
 fn llm(cmd: &str, prompt: &str) -> anyhow::Result<String> {
+    llm_with(cmd, prompt, &[])
+}
+
+/// As `llm`, with hints for the backend in its environment (`SEED`,
+/// `FORMAT=json`, `TEMPERATURE`). A backend is free to ignore them.
+pub(crate) fn llm_with(cmd: &str, prompt: &str, env: &[(&str, &str)]) -> anyhow::Result<String> {
     let mut child = Command::new("sh")
         .arg("-c")
         .arg(cmd)
+        .envs(env.iter().copied())
         .stdin(Stdio::piped())
         .stdout(Stdio::piped())
         .stderr(Stdio::inherit())
diff --git a/src/simulate.rs b/src/simulate.rs
new file mode 100644
index 0000000..39372a6
--- /dev/null
+++ b/src/simulate.rs
@@ -0,0 +1,554 @@
+//! Plays the whole business, many times.
+//!
+//! Input is the model `question_lint --needs-sim` exports. Each case is
+//! one persona on one run. A visitor model picks an alternative and
+//! fills the form in that person's words, in a manner that varies by
+//! run; the record lands in its bucket. Then the case is walked along
+//! the handling group's own desk buttons to a finished state - a
+//! different one each run, so every state and button gets exercised -
+//! and at each button a handler model is asked the one thing a model
+//! can judge: with only what the form collected, could this role
+//! really do what the button claims? If not, its question goes back to
+//! the visitor, who may or may not know the answer.
+//!
+//! What comes out is about the content, never the market:
+//! - a **misroute** is wording;
+//! - an **abandoned** form asks too much of that person;
+//! - an **ask-back** at (state, button) means the form did not collect
+//!   what the role needs there, and the question names the field;
+//! - an **unanswerable** ask-back is a fact the visitor never had, so
+//!   the process itself has to supply it;
+//! - **no route** means a finished state the desk cannot reach.
+//!
+//! Who wins a job is not simulated. That comes from the live bucket:
+//! the `success` states in needs.yaml are what a report over real
+//! records counts.
+
+use std::collections::BTreeMap;
+use std::io::Write as _;
+
+use anyhow::{anyhow, Context};
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Value};
+
+use crate::llm_with;
+
+const TRAITS: &[&str] = &[
+    "is precise and businesslike",
+    "types two short lines on a phone and skips anything optional",
+    "explains at length, with every detail they can think of",
+    "is unsure of the right terms and describes things loosely",
+    "is in a hurry and a little impatient",
+];
+
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+#[serde(tag = "outcome", content = "detail", rename_all = "snake_case")]
+pub enum Outcome {
+    Finished(String),
+    Stuck(String),
+    Misrouted(String),
+    Abandoned(String),
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct AskBack {
+    pub state: String,
+    pub button: String,
+    pub question: String,
+    /// The visitor knew the answer. When false, no form field would
+    /// have helped: the process has to find it out another way.
+    pub answerable: bool,
+}
+
+#[derive(Debug, Serialize)]
+struct Case {
+    persona: String,
+    need: String,
+    run: usize,
+    target: String,
+    record: BTreeMap<String, Value>,
+    states: Vec<String>,
+    /// "state -> button", in the order pressed.
+    pressed: Vec<String>,
+    touches: usize,
+    ask_backs: Vec<AskBack>,
+    outcome: Outcome,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct NeedScore {
+    pub cases: usize,
+    pub finished: usize,
+    pub stuck: usize,
+    pub misrouted: usize,
+    pub abandoned: usize,
+    pub touches: usize,
+    pub ask_backs: usize,
+    pub unanswerable: usize,
+    pub asked: Vec<AskBack>,
+    /// "state -> button": (times pressed, times the role had to ask first)
+    pub buttons: BTreeMap<String, (usize, usize)>,
+    pub stuck_at: Vec<String>,
+    pub misroutes: Vec<String>,
+    pub abandon_reasons: Vec<String>,
+}
+
+pub fn run(argv: &[String]) -> anyhow::Result<()> {
+    let value = |flag: &str| {
+        argv.iter()
+            .position(|a| a == flag)
+            .and_then(|i| argv.get(i + 1))
+            .cloned()
+    };
+    let usage = "usage: portal-trainer simulate --model <model.json> --out <dir> [--llm 'cmd'] [--handler-llm 'cmd'] [--runs 3] [--need <id>]";
+    let model_path = value("--model").ok_or_else(|| anyhow!(usage))?;
+    let out = std::path::PathBuf::from(value("--out").ok_or_else(|| anyhow!(usage))?);
+    let visitor_cmd = value("--llm").unwrap_or_else(|| "claude -p".to_string());
+    let handler_cmd = value("--handler-llm").unwrap_or_else(|| visitor_cmd.clone());
+    let runs: usize = value("--runs").map(|v| v.parse()).transpose()?.unwrap_or(3);
+    let only = value("--need");
+
+    let model: Value = serde_json::from_str(
+        &std::fs::read_to_string(&model_path).with_context(|| format!("reading {model_path}"))?,
+    )?;
+    std::fs::create_dir_all(&out)?;
+    let mut cases_file = std::fs::File::create(out.join("cases.jsonl"))?;
+    let mut scores: BTreeMap<String, NeedScore> = BTreeMap::new();
+
+    for persona in model["personas"].as_array().into_iter().flatten() {
+        let need_id = persona["need"].as_str().unwrap_or_default();
+        if only.as_deref().is_some_and(|o| o != need_id) {
+            continue;
+        }
+        let Some(need) = model["needs"]
+            .as_array()
+            .into_iter()
+            .flatten()
+            .find(|n| n["id"] == need_id)
+        else {
+            continue;
+        };
+        for run in 0..runs {
+            let case = play(&model, need, persona, run, &visitor_cmd, &handler_cmd)?;
+            eprintln!(
+                "{} #{run}: {:?} after {} touch(es), {} ask-back(s)",
+                case.persona,
+                case.outcome,
+                case.touches,
+                case.ask_backs.len()
+            );
+            tally(scores.entry(need_id.to_string()).or_default(), &case);
+            writeln!(cases_file, "{}", serde_json::to_string(&case)?)?;
+        }
+    }
+    std::fs::write(out.join("score.json"), serde_json::to_string_pretty(&scores)?)?;
+    print_scores(&scores);
+    Ok(())
+}
+
+fn play(
+    model: &Value,
+    need: &Value,
+    persona: &Value,
+    run: usize,
+    visitor_cmd: &str,
+    handler_cmd: &str,
+) -> anyhow::Result<Case> {
+    let who = persona["text"].as_str().unwrap_or_default();
+    let manner = TRAITS[run % TRAITS.len()];
+    let seed = |step: usize| (run * 1000 + step).to_string();
+    let mut case = Case {
+        persona: persona["id"].as_str().unwrap_or_default().to_string(),
+        need: need["id"].as_str().unwrap_or_default().to_string(),
+        run,
+        target: String::new(),
+        record: BTreeMap::new(),
+        states: Vec::new(),
+        pressed: Vec::new(),
+        touches: 0,
+        ask_backs: Vec::new(),
+        outcome: Outcome::Stuck("never started".to_string()),
+    };
+
+    // The visitor's side.
+    for (i, step) in need["steps"].as_array().into_iter().flatten().enumerate() {
+        let options = step["options"].as_array().cloned().unwrap_or_default();
+        if options.len() > 1 {
+            let prompt = format!(
+                "You are this person: {who}\n\nYou open a web page that asks: {}\nIt offers:\n{}\nWhich one do you pick? Judge only from your own situation and the wording.\nReply as JSON: {{\"choice\": \"<option heading, copied exactly>\"}}",
+                step["question"],
+                options
+                    .iter()
+                    .map(|o| format!("- {}: {}", o["name"].as_str().unwrap_or_default(), o["description"].as_str().unwrap_or_default()))
+                    .collect::<Vec<_>>()
+                    .join("\n")
+            );
+            let reply = ask(visitor_cmd, &prompt, &seed(i * 10))?;
+            let names: Vec<&str> = options.iter().filter_map(|o| o["name"].as_str()).collect();
+            let choice = resolve_choice(&names, reply["choice"].as_str().unwrap_or_default());
+            let expected = step["expect"].as_array().cloned().unwrap_or_default();
+            if !expected.iter().any(|e| e.as_str() == Some(choice.as_str())) {
+                case.outcome = Outcome::Misrouted(choice);
+                return Ok(case);
+            }
+        }
+        let fields = step["fields"].as_array().cloned().unwrap_or_default();
+        if fields.is_empty() {
+            continue;
+        }
+        let prompt = format!(
+            "You are this person: {who}\nToday you are someone who {manner}.\n\nYou chose \"{}\" and now face this form:\n{}\n\nFill it in as yourself. Use only what you would actually know or have at hand; never invent precise facts a person like you would not know - say so in the field instead. Optional fields may be left out. If this form asks for something you would refuse to give, or is more effort than your situation is worth to you, abandon it.\nReply as JSON: {{\"abandon\": false, \"why\": \"\", \"values\": {{\"<field name>\": \"<what you type>\"}}}}",
+            step["alternative"].as_str().unwrap_or_default(),
+            fields
+                .iter()
+                .map(|f| format!(
+                    "- {} (name: {}, type: {}{}{})",
+                    f["label"].as_str().unwrap_or_default(),
+                    f["name"].as_str().unwrap_or_default(),
+                    f["type"].as_str().unwrap_or("text"),
+                    if f["optional"] == true { ", optional" } else { ", required" },
+                    match f["options"].as_array() {
+                        Some(o) if !o.is_empty() => format!(", one of {}", Value::Array(o.clone())),
+                        _ => String::new(),
+                    }
+                ))
+                .collect::<Vec<_>>()
+                .join("\n")
+        );
+        let reply = ask(visitor_cmd, &prompt, &seed(i * 10 + 1))?;
+        if reply["abandon"] == true {
+            case.outcome = Outcome::Abandoned(reply["why"].as_str().unwrap_or_default().to_string());
+            return Ok(case);
+        }
+        if let Some(values) = reply["values"].as_object() {
+            for (k, v) in values {
+                case.record.insert(k.clone(), v.clone());
+            }
+        }
+    }
+
+    // The business's side. A model is a poor judge of who wins a job,
+    // and an 8B one decides by the tone of its prompt (measured: every
+    // case "won", then, one sentence later, every case "lost"). So the
+    // route is scripted, not decided: run by run, cases are sent to
+    // each finished state in turn, along the desk's own buttons, which
+    // covers every state and button of the machine. At each button the
+    // model is asked only what it can judge: with what the form
+    // collected, could this role really do what the button claims?
+    let bucket_name = need["lands_in"].as_str().unwrap_or_default();
+    let bucket = &model["buckets"][bucket_name];
+    let group = need["handled_by"].as_str().unwrap_or_default();
+    let done: Vec<&str> = need["done_when"].as_array().into_iter().flatten().filter_map(Value::as_str).collect();
+    let initial = bucket["initial"].as_str().unwrap_or("open").to_string();
+    let target = done.get(run % done.len().max(1)).copied().unwrap_or_default().to_string();
+    case.target = target.clone();
+    case.states.push(initial.clone());
+    let Some(route) = route_to(bucket, group, &initial, &target) else {
+        case.outcome = Outcome::Stuck(format!("no route of desk buttons from {initial} to {target}"));
+        return Ok(case);
+    };
+    for (step, (from, label, to)) in route.iter().enumerate() {
+        let prompt = format!(
+            "You work at a business as one of its {group}. A case in `{bucket_name}` is in state `{from}`. It came from {}, who wants {}.\n\nThis is everything you have about it - what the form collected, plus any follow-up:\n{}\n\nThe next thing that happens to this case is the desk button \"{label}\", which moves it to `{to}`. Pressing it claims that this has really been done.\n\nUsing only what is in front of you, could you really do that now? Answer false only when a fact that ONLY THE PERSON can give is missing and you could not proceed without it. Facts you find out yourself while doing the work do not count: you learn a building's condition by visiting it, a firm's registration by looking it up.\nReply as JSON: {{\"can_act\": true or false, \"need_to_know\": \"<the one question you would send them>\" or null}}",
+            need["who"].as_str().unwrap_or_default(),
+            need["wants"].as_str().unwrap_or_default(),
+            serde_json::to_string_pretty(&case.record)?,
+        );
+        let reply = ask(handler_cmd, &prompt, &seed(100 + step))?;
+        case.touches += 1;
+        let question = reply["need_to_know"].as_str().unwrap_or_default().trim().to_string();
+        if reply["can_act"] == false && !question.is_empty() {
+            let answer = ask(
+                visitor_cmd,
+                &format!(
+                    "You are this person: {who}\nToday you are someone who {manner}.\n\nThe business you wrote to comes back with a question: {question}\nAnswer as yourself, briefly. Use only what a person like you would really know.\nReply as JSON: {{\"known\": true or false, \"answer\": \"...\"}}"
+                ),
+                &seed(200 + step),
+            )?;
+            let n = case.ask_backs.len() + 1;
+            case.record.insert(format!("follow_up_{n}"), json!({ "asked": question, "answered": answer["answer"] }));
+            case.ask_backs.push(AskBack {
+                state: from.clone(),
+                button: label.clone(),
+                question,
+                answerable: answer["known"] != false,
+            });
+            case.touches += 1;
+        }
+        case.pressed.push(format!("{from} -> {label}"));
+        case.states.push(to.clone());
+    }
+    case.outcome = Outcome::Finished(target);
+    Ok(case)
+}
+
+/// The shortest chain of this group's desk buttons from one state to
+/// another, as (from, button label, to).
+fn route_to(bucket: &Value, group: &str, from: &str, to: &str) -> Option<Vec<(String, String, String)>> {
+    let moves: Vec<(String, String, String)> = bucket["desks"]
+        .as_array()
+        .into_iter()
+        .flatten()
+        .filter(|d| d["group"] == group)
+        .flat_map(|d| d["moves"].as_array().cloned().unwrap_or_default())
+        .filter_map(|m| Some((m["from"].as_str()?.to_string(), m["label"].as_str()?.to_string(), m["to"].as_str()?.to_string())))
+        .filter(|(f, _, t)| bucket["transitions"][f].as_array().is_some_and(|a| a.iter().any(|x| x == t.as_str())))
+        .collect();
+    let mut queue = std::collections::VecDeque::from([(from.to_string(), Vec::new())]);
+    let mut seen = std::collections::HashSet::from([from.to_string()]);
+    while let Some((state, path)) = queue.pop_front() {
+        if state == to {
+            return Some(path);
+        }
+        for (f, label, t) in moves.iter().filter(|(f, _, _)| *f == state) {
+            if seen.insert(t.clone()) {
+                let mut next = path.clone();
+                next.push((f.clone(), label.clone(), t.clone()));
+                queue.push_back((t.clone(), next));
+            }
+        }
+    }
+    None
+}
+
+/// Models often answer "Heading: its description" or change the case.
+/// The pick is the longest option heading the answer starts with;
+/// anything else is kept verbatim and counts as a misroute.
+fn resolve_choice(names: &[&str], raw: &str) -> String {
+    let answer = raw.trim().trim_matches('"').to_lowercase();
+    names
+        .iter()
+        .filter(|n| answer.starts_with(&n.to_lowercase()))
+        .max_by_key(|n| n.len())
+        .map(|n| n.to_string())
+        .unwrap_or_else(|| raw.trim().to_string())
+}
+
+/// One JSON-answering call. The backend is asked for JSON where it can
+/// be (`FORMAT=json`, honoured by engines/ollama.sh) and the first
+/// object in the reply is taken either way.
+fn ask(cmd: &str, prompt: &str, seed: &str) -> anyhow::Result<Value> {
+    let reply = llm_with(cmd, prompt, &[("FORMAT", "json"), ("SEED", seed), ("TEMPERATURE", "0.7")])?;
+    Ok(first_object(&reply).unwrap_or(Value::Null))
+}
+
+pub(crate) fn first_object(text: &str) -> Option<Value> {
+    let start = text.find('{')?;
+    let mut depth = 0usize;
+    let mut in_string = false;
+    let mut escaped = false;
+    for (i, c) in text[start..].char_indices() {
+        match (in_string, escaped, c) {
+            (true, true, _) => escaped = false,
+            (true, false, '\\') => escaped = true,
+            (true, false, '"') => in_string = false,
+            (false, _, '"') => in_string = true,
+            (false, _, '{') => depth += 1,
+            (false, _, '}') => {
+                depth -= 1;
+                if depth == 0 {
+                    return serde_json::from_str(&text[start..start + i + 1]).ok();
+                }
+            }
+            _ => {}
+        }
+    }
+    None
+}
+
+fn tally(score: &mut NeedScore, case: &Case) {
+    score.cases += 1;
+    score.touches += case.touches;
+    score.ask_backs += case.ask_backs.len();
+    score.unanswerable += case.ask_backs.iter().filter(|a| !a.answerable).count();
+    score.asked.extend(case.ask_backs.iter().cloned());
+    for button in &case.pressed {
+        score.buttons.entry(button.clone()).or_default().0 += 1;
+    }
+    for a in &case.ask_backs {
+        score.buttons.entry(format!("{} -> {}", a.state, a.button)).or_default().1 += 1;
+    }
+    match &case.outcome {
+        Outcome::Finished(_) => score.finished += 1,
+        Outcome::Stuck(at) => {
+            score.stuck += 1;
+            score.stuck_at.push(at.clone());
+        }
+        Outcome::Misrouted(picked) => {
+            score.misrouted += 1;
+            score.misroutes.push(format!("{}: {picked}", case.persona));
+        }
+        Outcome::Abandoned(why) => {
+            score.abandoned += 1;
+            score.abandon_reasons.push(format!("{}: {why}", case.persona));
+        }
+    }
+}
+
+fn pct(n: usize, of: usize) -> String {
+    if of == 0 {
+        "-".to_string()
+    } else {
+        format!("{:.0}%", n as f64 * 100.0 / of as f64)
+    }
+}
+
+fn per_case(n: usize, s: &NeedScore) -> String {
+    let reached = s.cases - s.misrouted - s.abandoned;
+    if reached == 0 {
+        "-".to_string()
+    } else {
+        format!("{:.1}", n as f64 / reached as f64)
+    }
+}
+
+fn print_scores(scores: &BTreeMap<String, NeedScore>) {
+    println!(
+        "\n{:<28} {:>5} {:>9} {:>9} {:>10} {:>9} {:>8} {:>10} {:>13}",
+        "need", "cases", "finished", "misroute", "abandoned", "no route", "touches", "ask-backs", "unanswerable"
+    );
+    for (need, s) in scores {
+        println!(
+            "{:<28} {:>5} {:>9} {:>9} {:>10} {:>9} {:>8} {:>10} {:>13}",
+            need,
+            s.cases,
+            pct(s.finished, s.cases),
+            pct(s.misrouted, s.cases),
+            pct(s.abandoned, s.cases),
+            pct(s.stuck, s.cases),
+            per_case(s.touches, s),
+            per_case(s.ask_backs, s),
+            per_case(s.unanswerable, s),
+        );
+    }
+    println!("\nWhere the role could not act on what the form collected:");
+    for (need, s) in scores {
+        for (button, (pressed, asked)) in s.buttons.iter().filter(|(_, (_, asked))| *asked > 0) {
+            println!("  {need}: {button} - had to ask first in {asked} of {pressed}");
+        }
+    }
+    for (need, s) in scores {
+        for a in &s.asked {
+            println!(
+                "  {need} [{} / {}]{} {}",
+                a.state,
+                a.button,
+                if a.answerable { "" } else { " (visitor did not know)" },
+                a.question
+            );
+        }
+        for at in &s.stuck_at {
+            println!("  {need}: {at}");
+        }
+        for m in &s.misroutes {
+            println!("  {need}: misrouted - {m}");
+        }
+        for r in &s.abandon_reasons {
+            println!("  {need}: abandoned - {r}");
+        }
+    }
+}
+
+/// `compare <a/score.json> <b/score.json>`: the same seeded cases
+/// against two versions of a site.
+pub fn compare(argv: &[String]) -> anyhow::Result<()> {
+    let (a, b) = match (argv.get(1), argv.get(2)) {
+        (Some(a), Some(b)) => (a, b),
+        _ => return Err(anyhow!("usage: portal-trainer compare <a/score.json> <b/score.json>")),
+    };
+    let load = |p: &String| -> anyhow::Result<BTreeMap<String, NeedScore>> {
+        Ok(serde_json::from_str(&std::fs::read_to_string(p).with_context(|| format!("reading {p}"))?)?)
+    };
+    let (a, b) = (load(a)?, load(b)?);
+    println!(
+        "{:<28} {:>14} {:>14} {:>14} {:>16} {:>16}",
+        "need", "misroute", "abandoned", "no route", "touches / case", "ask-backs / case"
+    );
+    for (need, sa) in &a {
+        let Some(sb) = b.get(need) else { continue };
+        println!(
+            "{:<28} {:>14} {:>14} {:>14} {:>16} {:>16}",
+            need,
+            format!("{} -> {}", pct(sa.misrouted, sa.cases), pct(sb.misrouted, sb.cases)),
+            format!("{} -> {}", pct(sa.abandoned, sa.cases), pct(sb.abandoned, sb.cases)),
+            format!("{} -> {}", pct(sa.stuck, sa.cases), pct(sb.stuck, sb.cases)),
+            format!("{} -> {}", per_case(sa.touches, sa), per_case(sb.touches, sb)),
+            format!("{} -> {}", per_case(sa.ask_backs, sa), per_case(sb.ask_backs, sb)),
+        );
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_first_json_object_is_found_inside_chatter() {
+        let v = first_object("Sure! {\"choice\": \"A {b}\", \"n\": {\"x\": 1}} trailing").unwrap();
+        assert_eq!(v["choice"], "A {b}");
+        assert_eq!(v["n"]["x"], 1);
+        assert!(first_object("no json").is_none());
+    }
+
+    #[test]
+    fn a_choice_is_matched_by_its_heading_however_the_model_dresses_it() {
+        let names = ["Ask about working", "Ask about working with us", "Get a price"];
+        assert_eq!(resolve_choice(&names, "Ask about working with us: For carpenters who..."), "Ask about working with us");
+        assert_eq!(resolve_choice(&names, " get a PRICE "), "Get a price");
+        assert_eq!(resolve_choice(&names, "Something else"), "Something else");
+    }
+
+    fn bucket() -> Value {
+        json!({
+            "initial": "open",
+            "transitions": {"open": ["surveyed", "lost"], "surveyed": ["quoted", "lost"], "quoted": ["won", "lost"]},
+            "desks": [
+                {"group": "pm", "moves": [
+                    {"from": "open", "to": "surveyed", "label": "Looked"},
+                    {"from": "surveyed", "to": "quoted", "label": "Quote sent"},
+                    {"from": "quoted", "to": "won", "label": "Won"},
+                    {"from": "open", "to": "lost", "label": "Not for us"},
+                    // Offered by the desk, refused by the machine.
+                    {"from": "open", "to": "won", "label": "Shortcut"}
+                ]},
+                {"group": "other", "moves": [{"from": "open", "to": "quoted", "label": "Theirs"}]}
+            ]
+        })
+    }
+
+    #[test]
+    fn routes_follow_this_groups_legal_buttons_only() {
+        let route = route_to(&bucket(), "pm", "open", "won").unwrap();
+        let labels: Vec<&str> = route.iter().map(|(_, l, _)| l.as_str()).collect();
+        assert_eq!(labels, ["Looked", "Quote sent", "Won"]);
+        assert_eq!(route_to(&bucket(), "pm", "open", "lost").unwrap().len(), 1);
+        assert!(route_to(&bucket(), "nobody", "open", "won").is_none());
+        assert_eq!(route_to(&bucket(), "pm", "open", "open").unwrap().len(), 0);
+    }
+
+    #[test]
+    fn a_case_is_tallied_by_how_it_ended_and_what_had_to_be_asked() {
+        let mut score = NeedScore::default();
+        let base = |outcome| Case {
+            persona: "p".into(),
+            need: "n".into(),
+            run: 0,
+            target: "won".into(),
+            record: BTreeMap::new(),
+            states: vec![],
+            pressed: vec!["open -> Looked".into(), "surveyed -> Quote sent".into()],
+            touches: 3,
+            ask_backs: vec![
+                AskBack { state: "open".into(), button: "Looked".into(), question: "Which floor?".into(), answerable: true },
+                AskBack { state: "surveyed".into(), button: "Quote sent".into(), question: "Load-bearing?".into(), answerable: false },
+            ],
+            outcome,
+        };
+        tally(&mut score, &base(Outcome::Finished("won".into())));
+        tally(&mut score, &base(Outcome::Stuck("no route".into())));
+        tally(&mut score, &base(Outcome::Misrouted("Wrong door".into())));
+        assert_eq!((score.cases, score.finished, score.stuck, score.misrouted), (3, 1, 1, 1));
+        assert_eq!((score.ask_backs, score.unanswerable), (6, 3));
+    }
+}