portal/src/trainer/score.rs

//! `score`: one command for the whole grading of a built site, which
//! every run under `runs/` used to carry as its own `score.sh`. The
//! lint (iris) writes the visitor tasks and the simulation model, the
//! blind visitor answers them with and without descriptions, the lint
//! scores the answers, and `rate` reads the cards. What comes out is
//! the two routing numbers, the misroutes, the rating table, and a
//! `summary.json` a run's README can quote.

use anyhow::{anyhow, bail, Context};
use std::path::{Path, PathBuf};
use std::process::Command;

pub fn run(argv: &[String]) -> anyhow::Result<()> {
    let flag = |name: &str| argv.iter().position(|a| a == name).and_then(|i| argv.get(i + 1)).cloned();
    let usage = "usage: portal-trainer score --site <content dir> --out <dir> [--iris <path>] [--llm 'cmd']";
    let site = PathBuf::from(flag("--site").ok_or_else(|| anyhow!(usage))?);
    let out = PathBuf::from(flag("--out").ok_or_else(|| anyhow!(usage))?);
    let iris = flag("--iris").unwrap_or_else(|| "iris".to_string());
    let cmd = flag("--llm").unwrap_or_else(|| "claude -p".to_string());
    let questions = site.join("questions");
    if !questions.is_dir() {
        bail!("{} has no questions/ directory", site.display());
    }
    std::fs::create_dir_all(&out)?;

    let tasks_full = out.join("tasks-full.jsonl");
    let tasks_skim = out.join("tasks-skim.jsonl");
    let model = out.join("model.json");
    let mut log = String::new();
    log.push_str(&lint(&iris, &questions, &["--needs-tasks", &s(&tasks_full), "--needs-sim", &s(&model)])?);
    log.push_str(&lint(&iris, &questions, &["--needs-tasks", &s(&tasks_skim), "--skim"])?);
    std::fs::write(out.join("iris.log"), &log)?;

    let mut summary = serde_json::Map::new();
    for (mode, tasks, skim) in [("full", &tasks_full, false), ("skim", &tasks_skim, true)] {
        eprintln!("score: the visitor answers {mode}");
        let answers = crate::trainer::answer_blind(&cmd, &std::fs::read_to_string(tasks)?, Some(&site), 0)?;
        let answers_path = out.join(format!("answers-{mode}.jsonl"));
        std::fs::write(&answers_path, &answers)?;
        let (tasks_arg, answers_arg) = (s(tasks), s(&answers_path));
        let mut args = vec!["--needs-tasks", tasks_arg.as_str(), "--needs-score", answers_arg.as_str()];
        if skim {
            args.push("--skim");
        }
        let report = lint(&iris, &questions, &args)?;
        println!("== {}", if skim { "headings only" } else { "with descriptions" });
        let mut right = None;
        for line in report.lines() {
            if line.starts_with("MISROUTE") || line.starts_with("HESITANT") || line.starts_with("UNANSWERED") {
                println!("{line}");
            }
            if let Some(rest) = line.strip_prefix("wording: ") {
                // "wording: 19/27 routed right (70%)"
                let mut parts = rest.split_whitespace().next().unwrap_or_default().split('/');
                let a: usize = parts.next().and_then(|x| x.parse().ok()).unwrap_or(0);
                let b: usize = parts.next().and_then(|x| x.parse().ok()).unwrap_or(0);
                right = Some((a, b));
                println!("{line}");
            }
        }
        let (a, b) = right.ok_or_else(|| anyhow!("iris printed no wording score for {mode}; its output:\n{report}"))?;
        summary.insert(mode.to_string(), serde_json::json!({ "right": a, "of": b }));
    }

    println!("== rate");
    let rated = out.join("rated");
    let rate_argv: Vec<String> = ["rate", "--model", &s(&model), "--out", &s(&rated), "--repo", &s(&site), "--llm", &cmd].iter().map(|x| x.to_string()).collect();
    crate::trainer::rate::rate(&rate_argv)?;

    summary.insert("site".into(), serde_json::json!(s(&site)));
    summary.insert("visitor".into(), serde_json::json!(cmd));
    std::fs::write(out.join("summary.json"), serde_json::to_string_pretty(&serde_json::Value::Object(summary.clone()))?)?;
    println!(
        "== summary: with descriptions {} of {}, headings only {} of {} (rate.json in {})",
        summary["full"]["right"], summary["full"]["of"], summary["skim"]["right"], summary["skim"]["of"], rated.display()
    );
    Ok(())
}

fn s(path: &Path) -> String {
    path.display().to_string()
}

/// `iris check --path <questions> <args>`; stdout and stderr together,
/// since the lint reports on both. A lint that fails is a site that
/// cannot be scored, and its output is the reason.
fn lint(iris: &str, questions: &Path, args: &[&str]) -> anyhow::Result<String> {
    let output = Command::new(iris)
        .arg("check")
        .arg("--path")
        .arg(questions)
        .args(args)
        .output()
        .with_context(|| format!("running {iris} (pass --iris <path> if it is not on PATH)"))?;
    let text = format!("{}{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr));
    if !output.status.success() {
        bail!("{iris} check --path {} {} failed:\n{text}", questions.display(), args.join(" "));
    }
    Ok(text)
}