diff --git a/engines/ollama.sh b/engines/ollama.sh index 707d60c..75bc0de 100755 --- a/engines/ollama.sh +++ b/engines/ollama.sh @@ -2,11 +2,16 @@ # LLM backend for portal-trainer on a local Ollama: prompt on stdin, # text on stdout. MODEL, OLLAMA_HOST, NUM_CTX and TIMEOUT override. # Thinking is off: the generator wants files, the answerer wants JSONL. +# The simulator sets SEED (so the same case replays the same way +# against two versions of a site), FORMAT=json and TEMPERATURE. set -eu jq -Rs --arg model "${MODEL:-qwen3:8b}" --argjson ctx "${NUM_CTX:-32768}" \ + --argjson temp "${TEMPERATURE:-0.2}" --arg seed "${SEED:-}" --arg format "${FORMAT:-}" \ '{model: $model, stream: false, think: false, - options: {num_ctx: $ctx, temperature: 0.2}, - messages: [{role: "user", content: .}]}' | + options: ({num_ctx: $ctx, temperature: $temp} + + (if $seed == "" then {} else {seed: ($seed | tonumber)} end)), + messages: [{role: "user", content: .}]} + + (if $format == "json" then {format: "json"} else {} end)' | curl -sS --fail-with-body --max-time "${TIMEOUT:-1800}" \ "${OLLAMA_HOST:-http://127.0.0.1:11434}/api/chat" \ -H "content-type: application/json" --data-binary @- | diff --git a/generator/reference.md b/generator/reference.md index 67554ff..b5afaf1 100644 --- a/generator/reference.md +++ b/generator/reference.md @@ -36,6 +36,12 @@ needs: done_when: [, ] # every state that counts as finished, good or bad max_steps: 1 # submissions before the record exists (1 or 2) max_fields: 4 # required fields the visitor will tolerate across the path + success: [] # the done_when states that are a good outcome for the business + stages: [open, , , , ] + # EVERY stage the business said a case goes through, first to last, + # read off their own sentences one verb at a time ("they come in, we + # go and look, we send a quote, we win or lose it" is five stages). + # The state machine must have a state for each, or the verifier fails. personas: - id: kebab-case need: diff --git a/src/main.rs b/src/main.rs index 3341eb9..761f3b0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,8 @@ use std::process::{Command, Stdio}; use anyhow::{anyhow, bail, Context}; +mod simulate; + const REFERENCE: &str = include_str!("../generator/reference.md"); struct Opts { @@ -38,6 +40,11 @@ fn main() -> anyhow::Result<()> { // for grading an existing site with any backend. Answers go to // stdout, ready for `question_lint --needs-score`. let argv: Vec = std::env::args().skip(1).collect(); + match argv.first().map(String::as_str) { + Some("simulate") => return simulate::run(&argv), + Some("compare") => return simulate::compare(&argv), + _ => {} + } if argv.first().map(String::as_str) == Some("answer") { let value = |flag: &str| { argv.iter() @@ -209,9 +216,16 @@ fn parse_args() -> anyhow::Result { /// Prompt on stdin, text on stdout. fn llm(cmd: &str, prompt: &str) -> anyhow::Result { + llm_with(cmd, prompt, &[]) +} + +/// As `llm`, with hints for the backend in its environment (`SEED`, +/// `FORMAT=json`, `TEMPERATURE`). A backend is free to ignore them. +pub(crate) fn llm_with(cmd: &str, prompt: &str, env: &[(&str, &str)]) -> anyhow::Result { let mut child = Command::new("sh") .arg("-c") .arg(cmd) + .envs(env.iter().copied()) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::inherit()) diff --git a/src/simulate.rs b/src/simulate.rs new file mode 100644 index 0000000..39372a6 --- /dev/null +++ b/src/simulate.rs @@ -0,0 +1,554 @@ +//! Plays the whole business, many times. +//! +//! Input is the model `question_lint --needs-sim` exports. Each case is +//! one persona on one run. A visitor model picks an alternative and +//! fills the form in that person's words, in a manner that varies by +//! run; the record lands in its bucket. Then the case is walked along +//! the handling group's own desk buttons to a finished state - a +//! different one each run, so every state and button gets exercised - +//! and at each button a handler model is asked the one thing a model +//! can judge: with only what the form collected, could this role +//! really do what the button claims? If not, its question goes back to +//! the visitor, who may or may not know the answer. +//! +//! What comes out is about the content, never the market: +//! - a **misroute** is wording; +//! - an **abandoned** form asks too much of that person; +//! - an **ask-back** at (state, button) means the form did not collect +//! what the role needs there, and the question names the field; +//! - an **unanswerable** ask-back is a fact the visitor never had, so +//! the process itself has to supply it; +//! - **no route** means a finished state the desk cannot reach. +//! +//! Who wins a job is not simulated. That comes from the live bucket: +//! the `success` states in needs.yaml are what a report over real +//! records counts. + +use std::collections::BTreeMap; +use std::io::Write as _; + +use anyhow::{anyhow, Context}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +use crate::llm_with; + +const TRAITS: &[&str] = &[ + "is precise and businesslike", + "types two short lines on a phone and skips anything optional", + "explains at length, with every detail they can think of", + "is unsure of the right terms and describes things loosely", + "is in a hurry and a little impatient", +]; + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +#[serde(tag = "outcome", content = "detail", rename_all = "snake_case")] +pub enum Outcome { + Finished(String), + Stuck(String), + Misrouted(String), + Abandoned(String), +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct AskBack { + pub state: String, + pub button: String, + pub question: String, + /// The visitor knew the answer. When false, no form field would + /// have helped: the process has to find it out another way. + pub answerable: bool, +} + +#[derive(Debug, Serialize)] +struct Case { + persona: String, + need: String, + run: usize, + target: String, + record: BTreeMap, + states: Vec, + /// "state -> button", in the order pressed. + pressed: Vec, + touches: usize, + ask_backs: Vec, + outcome: Outcome, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct NeedScore { + pub cases: usize, + pub finished: usize, + pub stuck: usize, + pub misrouted: usize, + pub abandoned: usize, + pub touches: usize, + pub ask_backs: usize, + pub unanswerable: usize, + pub asked: Vec, + /// "state -> button": (times pressed, times the role had to ask first) + pub buttons: BTreeMap, + pub stuck_at: Vec, + pub misroutes: Vec, + pub abandon_reasons: Vec, +} + +pub fn run(argv: &[String]) -> anyhow::Result<()> { + let value = |flag: &str| { + argv.iter() + .position(|a| a == flag) + .and_then(|i| argv.get(i + 1)) + .cloned() + }; + let usage = "usage: portal-trainer simulate --model --out [--llm 'cmd'] [--handler-llm 'cmd'] [--runs 3] [--need ]"; + let model_path = value("--model").ok_or_else(|| anyhow!(usage))?; + let out = std::path::PathBuf::from(value("--out").ok_or_else(|| anyhow!(usage))?); + let visitor_cmd = value("--llm").unwrap_or_else(|| "claude -p".to_string()); + let handler_cmd = value("--handler-llm").unwrap_or_else(|| visitor_cmd.clone()); + let runs: usize = value("--runs").map(|v| v.parse()).transpose()?.unwrap_or(3); + let only = value("--need"); + + let model: Value = serde_json::from_str( + &std::fs::read_to_string(&model_path).with_context(|| format!("reading {model_path}"))?, + )?; + std::fs::create_dir_all(&out)?; + let mut cases_file = std::fs::File::create(out.join("cases.jsonl"))?; + let mut scores: BTreeMap = BTreeMap::new(); + + for persona in model["personas"].as_array().into_iter().flatten() { + let need_id = persona["need"].as_str().unwrap_or_default(); + if only.as_deref().is_some_and(|o| o != need_id) { + continue; + } + let Some(need) = model["needs"] + .as_array() + .into_iter() + .flatten() + .find(|n| n["id"] == need_id) + else { + continue; + }; + for run in 0..runs { + let case = play(&model, need, persona, run, &visitor_cmd, &handler_cmd)?; + eprintln!( + "{} #{run}: {:?} after {} touch(es), {} ask-back(s)", + case.persona, + case.outcome, + case.touches, + case.ask_backs.len() + ); + tally(scores.entry(need_id.to_string()).or_default(), &case); + writeln!(cases_file, "{}", serde_json::to_string(&case)?)?; + } + } + std::fs::write(out.join("score.json"), serde_json::to_string_pretty(&scores)?)?; + print_scores(&scores); + Ok(()) +} + +fn play( + model: &Value, + need: &Value, + persona: &Value, + run: usize, + visitor_cmd: &str, + handler_cmd: &str, +) -> anyhow::Result { + let who = persona["text"].as_str().unwrap_or_default(); + let manner = TRAITS[run % TRAITS.len()]; + let seed = |step: usize| (run * 1000 + step).to_string(); + let mut case = Case { + persona: persona["id"].as_str().unwrap_or_default().to_string(), + need: need["id"].as_str().unwrap_or_default().to_string(), + run, + target: String::new(), + record: BTreeMap::new(), + states: Vec::new(), + pressed: Vec::new(), + touches: 0, + ask_backs: Vec::new(), + outcome: Outcome::Stuck("never started".to_string()), + }; + + // The visitor's side. + for (i, step) in need["steps"].as_array().into_iter().flatten().enumerate() { + let options = step["options"].as_array().cloned().unwrap_or_default(); + if options.len() > 1 { + let prompt = format!( + "You are this person: {who}\n\nYou open a web page that asks: {}\nIt offers:\n{}\nWhich one do you pick? Judge only from your own situation and the wording.\nReply as JSON: {{\"choice\": \"