//! Plays the whole business, many times. //! //! Input is the model `question_lint --needs-sim` exports. Each case is //! one persona on one run. A visitor model picks an alternative and //! fills the form in that person's words, in a manner that varies by //! run; the record lands in its bucket. Then the case is walked along //! the handling group's own desk buttons to a finished state - a //! different one each run, so every state and button gets exercised - //! and at each button a handler model is asked the one thing a model //! can judge: with only what the form collected, could this role //! really do what the button claims? If not, its question goes back to //! the visitor, who may or may not know the answer. //! //! What comes out is about the content, never the market: //! - a **misroute** is wording; //! - an **abandoned** form asks too much of that person; //! - an **ask-back** at (state, button) means the form did not collect //! what the role needs there, and the question names the field; //! - an **unanswerable** ask-back is a fact the visitor never had, so //! the process itself has to supply it; //! - **no route** means a finished state the desk cannot reach. //! //! Who wins a job is not simulated. That comes from the live bucket: //! the `success` states in needs.yaml are what a report over real //! records counts. use std::collections::BTreeMap; use std::io::Write as _; use anyhow::{anyhow, Context}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use crate::trainer::llm_with; const TRAITS: &[&str] = &[ "is precise and businesslike", "types two short lines on a phone and skips anything optional", "explains at length, with every detail they can think of", "is unsure of the right terms and describes things loosely", "is in a hurry and a little impatient", ]; #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] #[serde(tag = "outcome", content = "detail", rename_all = "snake_case")] pub enum Outcome { Finished(String), Stuck(String), Misrouted(String), Abandoned(String), } #[derive(Debug, Serialize, Deserialize, Clone)] pub struct AskBack { pub state: String, pub button: String, pub question: String, /// The visitor knew the answer. When false, no form field would /// have helped: the process has to find it out another way. pub answerable: bool, } #[derive(Debug, Serialize)] struct Case { persona: String, need: String, run: usize, target: String, record: BTreeMap, states: Vec, /// "state -> button", in the order pressed. pressed: Vec, /// Who pressed each of them: a group, or `submitter`. moved_by: Vec, touches: usize, ask_backs: Vec, outcome: Outcome, } #[derive(Debug, Default, Serialize, Deserialize)] pub struct NeedScore { pub cases: usize, pub finished: usize, pub stuck: usize, pub misrouted: usize, pub abandoned: usize, pub touches: usize, pub ask_backs: usize, pub unanswerable: usize, pub asked: Vec, /// "state -> button": (times pressed, times the role had to ask first) pub buttons: BTreeMap, pub stuck_at: Vec, pub misroutes: Vec, pub abandon_reasons: Vec, } pub fn run(argv: &[String]) -> anyhow::Result<()> { let value = |flag: &str| { argv.iter() .position(|a| a == flag) .and_then(|i| argv.get(i + 1)) .cloned() }; let usage = "usage: portal-trainer simulate --model --out [--llm 'cmd'] [--handler-llm 'cmd'] [--runs 3] [--need ]"; let model_path = value("--model").ok_or_else(|| anyhow!(usage))?; let out = std::path::PathBuf::from(value("--out").ok_or_else(|| anyhow!(usage))?); let visitor_cmd = value("--llm").unwrap_or_else(|| "claude -p".to_string()); let handler_cmd = value("--handler-llm").unwrap_or_else(|| visitor_cmd.clone()); let runs: usize = value("--runs").map(|v| v.parse()).transpose()?.unwrap_or(3); let only = value("--need"); let model: Value = serde_json::from_str( &std::fs::read_to_string(&model_path).with_context(|| format!("reading {model_path}"))?, )?; std::fs::create_dir_all(&out)?; let mut cases_file = std::fs::File::create(out.join("cases.jsonl"))?; let mut scores: BTreeMap = BTreeMap::new(); // A dry run asks no model anything: it walks every route of every // aggregate, so each button of each role is pressed once. Its cases // are what `needs_replay` puts through the real engine. if argv.iter().any(|a| a == "--dry") { for need in model["needs"].as_array().into_iter().flatten() { let id = need["id"].as_str().unwrap_or_default(); if only.as_deref().is_some_and(|o| o != id) { continue; } let bucket = &model["buckets"][need["lands_in"].as_str().unwrap_or_default()]; let done: Vec<&str> = need["done_when"] .as_array() .into_iter() .flatten() .filter_map(Value::as_str) .collect(); for (run, route) in all_routes(bucket, &done).into_iter().enumerate() { let mut states = vec![bucket["initial"].as_str().unwrap_or("open").to_string()]; states.extend(route.iter().map(|m| m.to.clone())); let case = Case { persona: "route".to_string(), need: id.to_string(), run, target: states.last().cloned().unwrap_or_default(), record: BTreeMap::from([("note".to_string(), json!("dry run"))]), pressed: route .iter() .map(|m| format!("{} -> {}", m.from, m.label)) .collect(), moved_by: route.iter().map(|m| m.by.clone()).collect(), touches: route.len(), ask_backs: Vec::new(), outcome: Outcome::Finished(states.last().cloned().unwrap_or_default()), states, }; tally(scores.entry(id.to_string()).or_default(), &case); writeln!(cases_file, "{}", serde_json::to_string(&case)?)?; } } std::fs::write( out.join("score.json"), serde_json::to_string_pretty(&scores)?, )?; let relays: usize = scores.values().map(|s| s.cases).sum(); println!( "dry run: {relays} route(s) across {} need(s) written to {}", scores.len(), out.join("cases.jsonl").display() ); return Ok(()); } for persona in model["personas"].as_array().into_iter().flatten() { let need_id = persona["need"].as_str().unwrap_or_default(); if only.as_deref().is_some_and(|o| o != need_id) { continue; } let Some(need) = model["needs"] .as_array() .into_iter() .flatten() .find(|n| n["id"] == need_id) else { continue; }; for run in 0..runs { let case = play(&model, need, persona, run, &visitor_cmd, &handler_cmd)?; eprintln!( "{} #{run}: {:?} after {} touch(es), {} ask-back(s)", case.persona, case.outcome, case.touches, case.ask_backs.len() ); tally(scores.entry(need_id.to_string()).or_default(), &case); writeln!(cases_file, "{}", serde_json::to_string(&case)?)?; } } std::fs::write( out.join("score.json"), serde_json::to_string_pretty(&scores)?, )?; print_scores(&scores); Ok(()) } fn play( model: &Value, need: &Value, persona: &Value, run: usize, visitor_cmd: &str, handler_cmd: &str, ) -> anyhow::Result { let who = persona["text"].as_str().unwrap_or_default(); let manner = TRAITS[run % TRAITS.len()]; let seed = |step: usize| (run * 1000 + step).to_string(); let mut case = Case { persona: persona["id"].as_str().unwrap_or_default().to_string(), need: need["id"].as_str().unwrap_or_default().to_string(), run, target: String::new(), record: BTreeMap::new(), states: Vec::new(), pressed: Vec::new(), moved_by: Vec::new(), touches: 0, ask_backs: Vec::new(), outcome: Outcome::Stuck("never started".to_string()), }; // The visitor's side. for (i, step) in need["steps"].as_array().into_iter().flatten().enumerate() { let options = step["options"].as_array().cloned().unwrap_or_default(); if options.len() > 1 { let prompt = format!( "You are this person: {who}\n\nYou open a web page that asks: {}\nIt offers:\n{}\nWhich one do you pick? Judge only from your own situation and the wording.\nReply as JSON: {{\"choice\": \"