From bf60879de9acc14f285edcb00a2b40b01323a579 Mon Sep 17 00:00:00 2001 From: Bendik Aagaard Lynghaug Date: Wed, 23 Sep 2026 20:25:55 +0200 Subject: [PATCH] score: the whole grading of a built site as one command iris writes the tasks and the model, the blind visitor answers with descriptions and with headings only, iris scores, rate reads the cards; the routing numbers land in summary.json. Replaces the score.sh every run carried. Proven on runs/uhhm-2026-09-23/site. Co-Authored-By: Claude Fable 5.1 --- README.md | 19 ++++++++-- src/main.rs | 4 ++- src/score.rs | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 src/score.rs diff --git a/README.md b/README.md index af146da..f69f897 100644 --- a/README.md +++ b/README.md @@ -269,10 +269,25 @@ and the scorers read all of it. you know where you are, who this is for, and what the button does. All four are controlled against the gutted twin. -## Rating wording +## Scoring a built site ``` -question_lint --path out/nordvik/questions --needs-sim model.json +portal-trainer score --site out/nordvik --out out/scored \ + --iris /var/local/cargo-target/release/iris \ + --llm "MODEL=qwen3:8b engines/ollama.sh" +``` + +One command for the whole grading, which every run under `runs/` +before 2026-09-23 carried as its own `score.sh`: iris writes the +visitor tasks and the simulation model, the blind visitor answers +them with descriptions and with headings only, iris scores the +answers (the misroutes are printed), and `rate` reads the cards. The +two routing numbers land in `summary.json`, the rating in +`rated/rate.json`. The pieces still run alone: `answer --tasks` for +one visitor model against tasks iris wrote, and: + +``` +iris check --path out/nordvik/questions --needs-sim model.json portal-trainer rate --model model.json --out out/rated --repo out/nordvik \ --llm "MODEL=qwen3:8b engines/ollama.sh" ``` diff --git a/src/main.rs b/src/main.rs index 33a1a8c..885b611 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,6 +26,7 @@ mod build; mod card; mod plan; mod rate; +mod score; mod simulate; const REFERENCE: &str = include_str!("../generator/reference.md"); @@ -54,6 +55,7 @@ fn main() -> anyhow::Result<()> { Some("build") => return build::run(&argv), Some("rate") => return rate::rate(&argv), Some("prefer") => return rate::prefer(&argv), + Some("score") => return score::run(&argv), _ => {} } if argv.first().map(String::as_str) == Some("answer") { @@ -445,7 +447,7 @@ fn dump(out: &Path) -> anyhow::Result { /// card portal renders: button, encouragement, sections with icon and /// guidance, fields with placeholders. Headings-only tasks (`--skim`, /// where the task's text is the heading itself) stay headings only. -fn answer_blind(cmd: &str, tasks: &str, repo: Option<&Path>) -> anyhow::Result { +pub(crate) fn answer_blind(cmd: &str, tasks: &str, repo: Option<&Path>) -> anyhow::Result { let mut answers = String::new(); for line in tasks.lines().filter(|l| !l.trim().is_empty()) { let task: serde_json::Value = serde_json::from_str(line)?; diff --git a/src/score.rs b/src/score.rs new file mode 100644 index 0000000..e5f801b --- /dev/null +++ b/src/score.rs @@ -0,0 +1,100 @@ +//! `score`: one command for the whole grading of a built site, which +//! every run under `runs/` used to carry as its own `score.sh`. The +//! lint (iris) writes the visitor tasks and the simulation model, the +//! blind visitor answers them with and without descriptions, the lint +//! scores the answers, and `rate` reads the cards. What comes out is +//! the two routing numbers, the misroutes, the rating table, and a +//! `summary.json` a run's README can quote. + +use anyhow::{anyhow, bail, Context}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +pub fn run(argv: &[String]) -> anyhow::Result<()> { + let flag = |name: &str| argv.iter().position(|a| a == name).and_then(|i| argv.get(i + 1)).cloned(); + let usage = "usage: portal-trainer score --site --out [--iris ] [--llm 'cmd']"; + let site = PathBuf::from(flag("--site").ok_or_else(|| anyhow!(usage))?); + let out = PathBuf::from(flag("--out").ok_or_else(|| anyhow!(usage))?); + let iris = flag("--iris").unwrap_or_else(|| "iris".to_string()); + let cmd = flag("--llm").unwrap_or_else(|| "claude -p".to_string()); + let questions = site.join("questions"); + if !questions.is_dir() { + bail!("{} has no questions/ directory", site.display()); + } + std::fs::create_dir_all(&out)?; + + let tasks_full = out.join("tasks-full.jsonl"); + let tasks_skim = out.join("tasks-skim.jsonl"); + let model = out.join("model.json"); + let mut log = String::new(); + log.push_str(&lint(&iris, &questions, &["--needs-tasks", &s(&tasks_full), "--needs-sim", &s(&model)])?); + log.push_str(&lint(&iris, &questions, &["--needs-tasks", &s(&tasks_skim), "--skim"])?); + std::fs::write(out.join("iris.log"), &log)?; + + let mut summary = serde_json::Map::new(); + for (mode, tasks, skim) in [("full", &tasks_full, false), ("skim", &tasks_skim, true)] { + eprintln!("score: the visitor answers {mode}"); + let answers = crate::answer_blind(&cmd, &std::fs::read_to_string(tasks)?, Some(&site))?; + let answers_path = out.join(format!("answers-{mode}.jsonl")); + std::fs::write(&answers_path, &answers)?; + let (tasks_arg, answers_arg) = (s(tasks), s(&answers_path)); + let mut args = vec!["--needs-tasks", tasks_arg.as_str(), "--needs-score", answers_arg.as_str()]; + if skim { + args.push("--skim"); + } + let report = lint(&iris, &questions, &args)?; + println!("== {}", if skim { "headings only" } else { "with descriptions" }); + let mut right = None; + for line in report.lines() { + if line.starts_with("MISROUTE") || line.starts_with("HESITANT") || line.starts_with("UNANSWERED") { + println!("{line}"); + } + if let Some(rest) = line.strip_prefix("wording: ") { + // "wording: 19/27 routed right (70%)" + let mut parts = rest.split_whitespace().next().unwrap_or_default().split('/'); + let a: usize = parts.next().and_then(|x| x.parse().ok()).unwrap_or(0); + let b: usize = parts.next().and_then(|x| x.parse().ok()).unwrap_or(0); + right = Some((a, b)); + println!("{line}"); + } + } + let (a, b) = right.ok_or_else(|| anyhow!("iris printed no wording score for {mode}; its output:\n{report}"))?; + summary.insert(mode.to_string(), serde_json::json!({ "right": a, "of": b })); + } + + println!("== rate"); + let rated = out.join("rated"); + let rate_argv: Vec = ["rate", "--model", &s(&model), "--out", &s(&rated), "--repo", &s(&site), "--llm", &cmd].iter().map(|x| x.to_string()).collect(); + crate::rate::rate(&rate_argv)?; + + summary.insert("site".into(), serde_json::json!(s(&site))); + summary.insert("visitor".into(), serde_json::json!(cmd)); + std::fs::write(out.join("summary.json"), serde_json::to_string_pretty(&serde_json::Value::Object(summary.clone()))?)?; + println!( + "== summary: with descriptions {} of {}, headings only {} of {} (rate.json in {})", + summary["full"]["right"], summary["full"]["of"], summary["skim"]["right"], summary["skim"]["of"], rated.display() + ); + Ok(()) +} + +fn s(path: &Path) -> String { + path.display().to_string() +} + +/// `iris check --path `; stdout and stderr together, +/// since the lint reports on both. A lint that fails is a site that +/// cannot be scored, and its output is the reason. +fn lint(iris: &str, questions: &Path, args: &[&str]) -> anyhow::Result { + let output = Command::new(iris) + .arg("check") + .arg("--path") + .arg(questions) + .args(args) + .output() + .with_context(|| format!("running {iris} (pass --iris if it is not on PATH)"))?; + let text = format!("{}{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); + if !output.status.success() { + bail!("{iris} check --path {} {} failed:\n{text}", questions.display(), args.join(" ")); + } + Ok(text) +}