portal/src/rate.rs

//! Understanding, readability, friendliness: three scores to iterate
//! wording against, each measured the way that keeps it from sticking
//! at its maximum.
//!
//! - **Readability** is counted, not judged: LIX, sentence length plus
//!   the share of words over six letters, over the page's prose only -
//!   the question, the descriptions and the followup - never headings
//!   or field labels, which are not sentences and would drag the score
//!   down to "very easy" whatever the prose says. Below 30 reads very
//!   easily, 30-40 easily, 40-50 takes effort, above 50 is official
//!   prose.
//! - **Understanding** is three yes-or-no questions put to each persona
//!   about the page in front of them: which option is theirs, what the
//!   form will ask, and what happens after sending. The last needs the
//!   followup page, which `rate` reads from the content repo (`--repo`)
//!   because the sim export stops at the form. `rate` also puts all
//!   three to a deliberately gutted twin of the page (headings replaced
//!   by "Option 1", descriptions, the form line and the followup
//!   removed). An instrument that still says yes to the twin is not
//!   measuring the page, and `rate` says so, per question. Every page
//!   on a need's path is rated: behind a door, "which" is decided on
//!   the door page and the form's page carries "asked" and "next".
//! - **Friendliness** is a preference between two versions, never an
//!   absolute mark: `prefer` shows each persona both versions in both
//!   orders; a pick that flips with the order counts as no preference.

use std::collections::BTreeMap;
use std::path::Path;

use anyhow::{anyhow, Context};
use serde_json::{json, Value};

use crate::card;
use crate::simulate::ask;

fn flag(argv: &[String], name: &str) -> Option<String> {
    argv.iter().position(|a| a == name).and_then(|i| argv.get(i + 1)).cloned()
}

fn load(path: &str) -> anyhow::Result<Value> {
    Ok(serde_json::from_str(&std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?)?)
}

/// LIX = words per sentence + 100 * long words / words.
pub fn lix(text: &str) -> f64 {
    let words: Vec<&str> = text
        .split(|c: char| !(c.is_alphanumeric() || c == '\'' || c == '-'))
        .filter(|w| w.chars().any(char::is_alphabetic))
        .collect();
    if words.is_empty() {
        return 0.0;
    }
    let sentences = text.split(['.', '!', '?', ':', '\n']).filter(|s| s.chars().any(char::is_alphabetic)).count().max(1);
    let long = words.iter().filter(|w| w.chars().count() > 6).count();
    words.len() as f64 / sentences as f64 + 100.0 * long as f64 / words.len() as f64
}

pub fn lix_band(score: f64) -> &'static str {
    match score {
        s if s < 30.0 => "very easy",
        s if s < 40.0 => "easy",
        s if s < 50.0 => "takes effort",
        _ => "official prose",
    }
}

/// The page as the visitor meets it: the question, every alternative,
/// the form behind the one that serves them, and the page they land on
/// after sending, when the repo is there to read it from.
fn page_text(step: &Value, page: Option<&Value>, followup: Option<&Value>, gutted: bool) -> String {
    let mut text = match (gutted, page) {
        (false, Some(p)) => card::page_head(p),
        _ => format!("{}\n", step["question"].as_str().unwrap_or_default()),
    };
    for (i, option) in step["options"].as_array().into_iter().flatten().enumerate() {
        if gutted {
            text.push_str(&format!("\n- Option {}\n", i + 1));
            continue;
        }
        // The whole card when the repo is there to read it from;
        // heading and description otherwise.
        let card = page
            .and_then(|p| p["alternatives"].as_array())
            .and_then(|alts| alts.iter().find(|a| a["name"] == option["name"]))
            .map(|a| card::alternative(a, true));
        match card {
            Some(c) => text.push_str(&format!("\n{c}")),
            None => text.push_str(&format!(
                "\n- {}\n  {}\n",
                option["name"].as_str().unwrap_or_default(),
                option["description"].as_str().unwrap_or_default()
            )),
        }
    }
    if page.is_some() && !gutted {
        // The cards carry their own "Asks for" lines.
        return finish_page_text(text, followup);
    }
    // The twin has no form line at all: "Field 1, Field 2" still
    // tells a reader how many things they will be asked, and a model
    // answers "can you tell what you will be asked" with yes to that
    // (measured: 2 of 2 on every need).
    let fields: Vec<&str> = step["fields"].as_array().into_iter().flatten().filter_map(|f| f["label"].as_str()).collect();
    if !gutted && !fields.is_empty() {
        text.push_str(&format!(
            "\nThe form under \"{}\" asks for: {}.\n",
            step["alternative"].as_str().unwrap_or_default(),
            fields.join(", ")
        ));
    }
    if gutted {
        return text;
    }
    finish_page_text(text, followup)
}

fn finish_page_text(mut text: String, followup: Option<&Value>) -> String {
    if let Some(next) = followup {
        text.push_str(&format!("\nAfter you send it, the next page says:\n{}\n", next["name"].as_str().unwrap_or_default()));
        if let Some(d) = next["description"].as_str().filter(|d| !d.trim().is_empty()) {
            text.push_str(&format!("{}\n", d.trim()));
        }
        for alt in next["alternatives"].as_array().into_iter().flatten() {
            let d = alt["description"].as_str().unwrap_or_default().trim();
            text.push_str(&format!("- {}{}{}\n", alt["name"].as_str().unwrap_or_default(), if d.is_empty() { "" } else { ": " }, d));
        }
    }
    text
}

/// The running text a reader has to parse: the question, each
/// description, and the followup. Headings and labels are left out.
fn prose_text(step: &Value, followup: Option<&Value>) -> String {
    let mut prose = vec![step["question"].as_str().unwrap_or_default().to_string()];
    prose.extend(step["options"].as_array().into_iter().flatten().filter_map(|o| o["description"].as_str()).map(str::to_string));
    if let Some(next) = followup {
        prose.extend(next["description"].as_str().map(str::to_string));
        prose.extend(next["alternatives"].as_array().into_iter().flatten().filter_map(|a| a["description"].as_str()).map(str::to_string));
    }
    prose.into_iter().filter(|p| !p.trim().is_empty()).collect::<Vec<_>>().join("\n")
}

/// The page a visitor lands on after the step's alternative: read from
/// the content repo, since the sim export stops at the form. `None`
/// when there is no repo, no `action`, or the page is not there.
fn followup_page(repo: Option<&Path>, step: &Value) -> Option<Value> {
    let repo = repo?;
    let page = load_page(repo, step["page"].as_str()?)?;
    let alternative = step["alternative"].as_str()?;
    let action = page["alternatives"].as_array()?.iter().find(|a| a["name"] == alternative)?["action"].as_str()?;
    load_page(repo, action)
}

/// `/` is `questions/index.yaml`; `/a/b` is `questions/a/b.yaml` or
/// `questions/a/b/index.yaml`.
pub(crate) fn load_page(repo: &Path, route: &str) -> Option<Value> {
    let rel = route.trim_matches('/');
    let candidates = if rel.is_empty() {
        vec![repo.join("questions/index.yaml")]
    } else {
        vec![repo.join(format!("questions/{rel}.yaml")), repo.join(format!("questions/{rel}/index.yaml"))]
    };
    let text = candidates.iter().find_map(|p| std::fs::read_to_string(p).ok())?;
    serde_yaml::from_str(&text).ok()
}

/// What one page may ask of one person before it should be split into
/// a further question, or the fields moved to a later stage of the
/// case where they are actually needed. Complexity belongs in the
/// number of steps and states, not in the weight of any one page.
const MAX_OPTIONS: usize = 5;
const MAX_REQUIRED: usize = 5;

/// (alternatives offered, required fields in the form that serves them)
fn page_load(step: &Value) -> (usize, usize) {
    (
        step["options"].as_array().map_or(0, Vec::len),
        step["fields"].as_array().into_iter().flatten().filter(|f| f["optional"] != true).count(),
    )
}

const QUESTIONS: [(&str, &str); 4] = [
    ("which", "Is it clear from the page which option is meant for someone in your situation?"),
    ("where", "Do you know where you are on this site, who this option is for, and what pressing its button will do?"),
    ("asked", "Can you tell from the page what you will be asked to provide?"),
    ("next", "Does the page tell you what happens after you send it, or who will read it?"),
];
const N: usize = QUESTIONS.len();

fn understanding(cmd: &str, who: &str, page: &str, seed: usize) -> anyhow::Result<[bool; N]> {
    let prompt = format!(
        "You are this person: {who}\n\nThis is the whole of a web page you have just opened:\n---\n{page}---\n\nAnswer four questions about it as yourself. Say yes only when the page text itself gives you that; if you would be guessing, say no.\n{}\nReply as JSON: {{\"which\": true or false, \"where\": true or false, \"asked\": true or false, \"next\": true or false}}",
        QUESTIONS.iter().map(|(k, q)| format!("- {k}: {q}")).collect::<Vec<_>>().join("\n")
    );
    let reply = ask(cmd, &prompt, &seed.to_string())?;
    Ok([reply["which"] == true, reply["where"] == true, reply["asked"] == true, reply["next"] == true])
}

pub fn rate(argv: &[String]) -> anyhow::Result<()> {
    let usage = "usage: portal-trainer rate --model <model.json> --out <dir> [--repo <content dir>] [--llm 'cmd']";
    let model = load(&flag(argv, "--model").ok_or_else(|| anyhow!(usage))?)?;
    let out = std::path::PathBuf::from(flag(argv, "--out").ok_or_else(|| anyhow!(usage))?);
    let repo = flag(argv, "--repo").map(std::path::PathBuf::from);
    let cmd = flag(argv, "--llm").unwrap_or_else(|| "claude -p".to_string());
    std::fs::create_dir_all(&out)?;
    if repo.is_none() {
        eprintln!("rate: no --repo, so no followup page is shown and \"next\" is not measured");
    }

    let mut report = BTreeMap::new();
    println!("{:<28} {:>6} {:<14} {:>8} {:>8} {:>8} {:>8}   {:<22}   {:<22}", "need", "LIX", "", "which", "where", "asked", "next", "gutted twin says", "page load");
    for need in model["needs"].as_array().into_iter().flatten() {
        let id = need["id"].as_str().unwrap_or_default();
        let steps = need["steps"].as_array().cloned().unwrap_or_default();
        let personas: Vec<&Value> = model["personas"].as_array().into_iter().flatten().filter(|p| p["need"] == id).collect();
        if personas.is_empty() {
            continue;
        }
        // Every page on the path is rated. The followup follows only
        // the last page's alternative: on a door page, what the
        // alternative leads to is the next page, not what happens
        // after sending.
        let mut pages = Vec::new();
        for (i, step) in steps.iter().enumerate() {
            let followup = if i + 1 == steps.len() { followup_page(repo.as_deref(), step) } else { None };
            let page = repo.as_deref().and_then(|r| load_page(r, step["page"].as_str().unwrap_or("/")));
            let row = if steps.len() > 1 { format!("{id} @{}", step["page"].as_str().unwrap_or_default()) } else { id.to_string() };
            let readability = lix(&prose_text(step, followup.as_ref()));
            let (mut yes, mut control) = ([0usize; N], [0usize; N]);
            for (i, persona) in personas.iter().enumerate() {
                let who = persona["text"].as_str().unwrap_or_default();
                for (tally, gutted) in [(&mut yes, false), (&mut control, true)] {
                    let answers = understanding(&cmd, who, &page_text(step, page.as_ref(), followup.as_ref(), gutted), i)?;
                    for (slot, answer) in tally.iter_mut().zip(answers) {
                        *slot += answer as usize;
                    }
                }
            }
            let n = personas.len();
            let measured = |k: usize| k != 3 || followup.is_some();
            let share = |k: usize| if measured(k) { format!("{}/{n}", yes[k]) } else { "-".to_string() };
            // The twin has no headings, no descriptions, no cards and no
            // followup: every question must collapse on it, or that
            // question is agreeing with anything.
            let control_ok: Vec<bool> = (0..N).map(|k| !measured(k) || control[k] * 2 < n.max(1)).collect();
            let failed: Vec<&str> = QUESTIONS.iter().zip(&control_ok).filter(|(_, ok)| !**ok).map(|((k, _), _)| *k).collect();
            let (options, required) = page_load(step);
            println!(
                "{:<28} {:>6.0} {:<14} {:>8} {:>8} {:>8} {:>8}   {:<22}   {:<22}",
                row,
                readability,
                lix_band(readability),
                share(0),
                share(1),
                share(2),
                share(3),
                format!(
                    "{}/{}/{}/{} {}",
                    control[0],
                    control[1],
                    control[2],
                    if measured(3) { control[3].to_string() } else { "-".to_string() },
                    if failed.is_empty() { "ok".to_string() } else { format!("FAILED: {}", failed.join(", ")) }
                ),
                format!(
                    "{options} options, {required} required{}",
                    if options > MAX_OPTIONS || required > MAX_REQUIRED { " - SPLIT" } else { "" }
                ),
            );
            pages.push(json!({
                "page": step["page"], "lix": readability, "personas": n,
                "options": options, "required_fields": required,
                "overloaded": options > MAX_OPTIONS || required > MAX_REQUIRED,
                "followup_shown": followup.is_some(),
                "understanding": {"which": yes[0], "where": yes[1], "asked": yes[2], "next": if measured(3) { json!(yes[3]) } else { Value::Null }},
                "control": {"which": control[0], "where": control[1], "asked": control[2], "next": if measured(3) { json!(control[3]) } else { Value::Null }},
                "control_ok": failed.is_empty(),
            }));
        }
        report.insert(id.to_string(), json!({ "steps": pages.len(), "pages": pages }));
    }
    std::fs::write(out.join("rate.json"), serde_json::to_string_pretty(&report)?)?;
    Ok(())
}

pub fn prefer(argv: &[String]) -> anyhow::Result<()> {
    let usage = "usage: portal-trainer prefer --a <model-a.json> --b <model-b.json> [--llm 'cmd']";
    let a = load(&flag(argv, "--a").ok_or_else(|| anyhow!(usage))?)?;
    let b = load(&flag(argv, "--b").ok_or_else(|| anyhow!(usage))?)?;
    let cmd = flag(argv, "--llm").unwrap_or_else(|| "claude -p".to_string());

    println!("{:<28} {:>5} {:>5} {:>14}   {:>5} {:>5} {:>14}   {:>9}", "need", "A", "B", "no preference", "A", "B", "no preference", "LIX A->B");
    println!("{:<28} {:^26}   {:^26}", "", "friendlier", "clearer");
    for need_a in a["needs"].as_array().into_iter().flatten() {
        let id = need_a["id"].as_str().unwrap_or_default();
        let Some(need_b) = b["needs"].as_array().into_iter().flatten().find(|n| n["id"] == id) else { continue };
        let (Some(step_a), Some(step_b)) = (need_a["steps"].as_array().and_then(|s| s.first()), need_b["steps"].as_array().and_then(|s| s.first())) else { continue };
        let (text_a, text_b) = (page_text(step_a, None, None, false), page_text(step_b, None, None, false));
        if text_a == text_b {
            continue;
        }
        // [friendlier, clearer] x [A, B, none]
        let mut wins = [[0usize; 3]; 2];
        for (i, persona) in a["personas"].as_array().into_iter().flatten().filter(|p| p["need"] == id).enumerate() {
            let who = persona["text"].as_str().unwrap_or_default();
            let mut picks: [[&str; 2]; 2] = [["", ""], ["", ""]];
            for (order, (first, second)) in [(&text_a, &text_b), (&text_b, &text_a)].into_iter().enumerate() {
                let prompt = format!(
                    "You are this person: {who}\n\nTwo versions of the same web page. They lead to the same thing; only the wording differs.\n\n=== VERSION 1 ===\n{first}\n=== VERSION 2 ===\n{second}\n\nAs yourself: which version makes you feel more welcome and at ease, and which one makes it clearer what to do?\nReply as JSON: {{\"friendlier\": 1 or 2, \"clearer\": 1 or 2}}"
                );
                let reply = ask(&cmd, &prompt, &(i * 2 + order).to_string())?;
                for (k, key) in ["friendlier", "clearer"].iter().enumerate() {
                    let first_won = reply[key] == 1;
                    // In the second order, version 1 is B.
                    picks[k][order] = if first_won == (order == 0) { "A" } else { "B" };
                }
            }
            for k in 0..2 {
                let slot = match picks[k] {
                    ["A", "A"] => 0,
                    ["B", "B"] => 1,
                    _ => 2,
                };
                wins[k][slot] += 1;
            }
        }
        println!(
            "{:<28} {:>5} {:>5} {:>14}   {:>5} {:>5} {:>14}   {:>9}",
            id, wins[0][0], wins[0][1], wins[0][2], wins[1][0], wins[1][1], wins[1][2],
            format!("{:.0}->{:.0}", lix(&text_a), lix(&text_b))
        );
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn lix_separates_plain_words_from_official_prose() {
        let plain = "Get someone to look at a building. We come and see it. Then we tell you what it takes.";
        let official = "Submission of documentation regarding rehabilitation requirements necessitates comprehensive structural evaluation procedures.";
        assert!(lix(plain) < 30.0, "{}", lix(plain));
        assert!(lix(official) > 60.0, "{}", lix(official));
        assert_eq!(lix_band(lix(plain)), "very easy");
        assert_eq!(lix(""), 0.0);
    }

    #[test]
    fn the_gutted_twin_keeps_the_question_and_loses_the_wording() {
        let step = json!({
            "question": "What brings you here?",
            "options": [{"name": "Get a price", "description": "For owners."}, {"name": "Work with us", "description": "For crews."}],
            "alternative": "Get a price",
            "fields": [{"label": "Email"}, {"label": "The build"}]
        });
        let next = json!({"name": "What happens now?", "description": "A project manager reads it.", "alternatives": [{"name": "We call you within two days", "description": ""}]});
        let page = page_text(&step, None, Some(&next), false);
        assert!(page.contains("Get a price") && page.contains("asks for: Email, The build"));
        assert!(page.contains("After you send it") && page.contains("A project manager reads it."));
        // With the repo's page in hand, each option is its whole card.
        let repo_page = json!({"name": "What brings you here?", "description": "", "alternatives": [
            {"name": "Get a price", "description": "For owners.", "consequence": ["Send it"], "encouragements": ["We answer within two days."], "features": [{"name": "Your brief", "requirements": [{"name": "email", "label": "Email"}]}]},
            {"name": "Work with us", "description": "For crews."}]});
        let rich = page_text(&step, Some(&repo_page), Some(&next), false);
        assert!(rich.contains("Button: Send it") && rich.contains("We answer within two days.") && rich.contains("Asks for: Email"));
        assert!(!rich.contains("asks for: Email, The build"));
        let twin = page_text(&step, Some(&repo_page), Some(&next), true);
        assert!(!twin.contains("Send it") && !twin.contains("Your brief"));
        let twin = page_text(&step, None, Some(&next), true);
        assert!(twin.contains("What brings you here?") && twin.contains("Option 2"));
        assert!(!twin.contains("asks for"));
        assert!(!twin.contains("Get a price") && !twin.contains("Email") && !twin.contains("project manager"));
        // Readability is measured on the sentences, not the headings.
        let prose = prose_text(&step, Some(&next));
        assert_eq!(prose, "What brings you here?\nFor owners.\nFor crews.\nA project manager reads it.");
    }

    #[test]
    fn the_followup_page_is_read_from_the_repo_by_following_the_action() {
        let dir = std::env::temp_dir().join(format!("portal-trainer-rate-{}", std::process::id()));
        std::fs::create_dir_all(dir.join("questions/received")).unwrap();
        std::fs::write(dir.join("questions/index.yaml"), "name: What?\nalternatives:\n  - name: Get a price\n    action: /received/price\n").unwrap();
        std::fs::write(dir.join("questions/received/price.yaml"), "name: What happens now?\ndescription: We read it.\nalternatives:\n  - name: Back\n").unwrap();
        let step = json!({"page": "/", "alternative": "Get a price"});
        let next = followup_page(Some(&dir), &step).unwrap();
        assert_eq!(next["name"], "What happens now?");
        assert!(followup_page(None, &step).is_none());
        assert!(followup_page(Some(&dir), &json!({"page": "/", "alternative": "Nope"})).is_none());
        std::fs::remove_dir_all(dir).ok();
    }
}