portal/src/trainer/report.rs

//! What a live site's storylines did, read back as counts.
//!
//! `portal-trainer report --site https://vel.example [--needs needs.yaml]`
//! fetches the site's `/report` (portal docs, Storylines): per bucket,
//! per storyline, per state, a count and nothing else. With the site's
//! `needs.yaml` the states that end well are known, and each storyline
//! is scored on how many cases it brought in and how many of those
//! ended well. Storylines are assigned evenly, so the counts compare
//! directly. This is the live half of what the scorer measures offline
//! with personas: the same tones, judged by what people did.
use std::collections::BTreeMap;

use anyhow::{anyhow, bail, Context};
use serde::Deserialize;

#[derive(Debug, Deserialize)]
struct Report {
    storylines: usize,
    /// bucket -> storyline -> state -> count
    buckets: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
}

#[derive(Debug, Default, Deserialize)]
struct Needs {
    #[serde(default)]
    needs: Vec<Need>,
    #[serde(default)]
    tones: Vec<String>,
}

#[derive(Debug, Deserialize)]
struct Need {
    lands_in: String,
    #[serde(default)]
    done_when: Vec<String>,
    #[serde(default)]
    success: Vec<String>,
}

/// One storyline's showing in one bucket.
#[derive(Debug, Default, PartialEq)]
struct Showing {
    arrived: u64,
    ended_well: u64,
}

/// Per bucket, per storyline index, how many arrived and how many
/// ended in a state the need calls success (or, without `success`,
/// any finished state).
fn showings(counts: &BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>, needs: &[Need]) -> BTreeMap<String, Vec<Showing>> {
    let mut out = BTreeMap::new();
    for (bucket, per_story) in counts {
        let well: Vec<&str> = needs
            .iter()
            .filter(|n| &n.lands_in == bucket)
            .flat_map(|n| if n.success.is_empty() { n.done_when.iter() } else { n.success.iter() })
            .map(String::as_str)
            .collect();
        let arity = per_story.keys().filter_map(|k| k.parse::<usize>().ok()).max().map(|m| m + 1).unwrap_or(1);
        let mut rows: Vec<Showing> = (0..arity).map(|_| Showing::default()).collect();
        for (story, states) in per_story {
            let Ok(n) = story.parse::<usize>() else { continue };
            for (state, count) in states {
                rows[n].arrived += count;
                if well.contains(&state.as_str()) {
                    rows[n].ended_well += count;
                }
            }
        }
        out.insert(bucket.clone(), rows);
    }
    out
}

pub fn run(argv: &[String]) -> anyhow::Result<()> {
    let value = |flag: &str| argv.iter().position(|a| a == flag).and_then(|i| argv.get(i + 1)).cloned();
    let site = value("--site").ok_or_else(|| anyhow!("usage: portal-trainer report --site <https://the.site> [--needs <needs.yaml>]"))?;
    let url = format!("{}/report", site.trim_end_matches('/'));
    let fetched = std::process::Command::new("curl").args(["-sSf", "--max-time", "30", &url]).output().context("running curl")?;
    if !fetched.status.success() {
        bail!("could not read {url}: {}", String::from_utf8_lossy(&fetched.stderr).trim());
    }
    let report: Report = serde_json::from_slice(&fetched.stdout).with_context(|| format!("reading {url}"))?;
    let needs: Needs = match value("--needs") {
        Some(path) => serde_yaml::from_str(&std::fs::read_to_string(&path).with_context(|| format!("reading {path}"))?)?,
        None => Needs::default(),
    };
    println!("{site}: {} storyline(s)", report.storylines);
    for (i, tone) in needs.tones.iter().enumerate() {
        println!("  [{i}] {tone}");
    }
    if report.buckets.is_empty() {
        println!("nothing sent in yet");
        return Ok(());
    }
    let rows = showings(&report.buckets, &needs.needs);
    for (bucket, per_story) in &rows {
        println!("\n{bucket}");
        for (n, showing) in per_story.iter().enumerate() {
            let rate = if showing.arrived > 0 { format!("{:.0}%", 100.0 * showing.ended_well as f64 / showing.arrived as f64) } else { "-".to_string() };
            println!("  storyline {n}: {} arrived, {} ended well ({rate})", showing.arrived, showing.ended_well);
        }
        if per_story.len() > 1 {
            let best = per_story.iter().enumerate().max_by_key(|(_, s)| (s.ended_well, s.arrived)).map(|(n, _)| n).unwrap_or(0);
            println!("  most so far: storyline {best}");
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn a_storyline_is_scored_on_what_arrived_and_what_ended_well() {
        let counts: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>> =
            serde_json::from_str(r#"{"hazards": {"0": {"open": 3, "fixed": 4}, "1": {"open": 1, "fixed": 9, "dropped": 2}}}"#).unwrap();
        let needs = vec![Need { lands_in: "hazards".into(), done_when: vec!["fixed".into(), "dropped".into()], success: vec!["fixed".into()] }];
        let rows = showings(&counts, &needs);
        assert_eq!(rows["hazards"][0], Showing { arrived: 7, ended_well: 4 });
        assert_eq!(rows["hazards"][1], Showing { arrived: 12, ended_well: 9 });
        // Without a success list, any finished state counts.
        let any = vec![Need { lands_in: "hazards".into(), done_when: vec!["fixed".into(), "dropped".into()], success: vec![] }];
        assert_eq!(showings(&counts, &any)["hazards"][1].ended_well, 11);
    }
}