254 lines
12 KiB
Rust
254 lines
12 KiB
Rust
|
//! Understanding, readability, friendliness: three scores to iterate
|
||
|
|
//! wording against, each measured the way that keeps it from sticking
|
||
|
|
//! at its maximum.
|
||
|
|
//!
|
||
|
|
//! - **Readability** is counted, not judged: LIX, sentence length plus
|
||
|
|
//! the share of words over six letters. Below 30 reads very easily,
|
||
|
|
//! 30-40 easily, 40-50 takes effort, above 50 is official prose.
|
||
|
|
//! - **Understanding** is three yes-or-no questions put to each persona
|
||
|
|
//! about the page in front of them. `rate` also puts them to a
|
||
|
|
//! deliberately gutted twin of the page (headings replaced by
|
||
|
|
//! "Option 1", descriptions removed). An instrument that still says
|
||
|
|
//! yes to the twin is not measuring the page, and `rate` says so.
|
||
|
|
//! - **Friendliness** is a preference between two versions, never an
|
||
|
|
//! absolute mark: `prefer` shows each persona both versions in both
|
||
|
|
//! orders; a pick that flips with the order counts as no preference.
|
||
|
|
|
||
|
|
use std::collections::BTreeMap;
|
||
|
|
|
||
|
|
use anyhow::{anyhow, Context};
|
||
|
|
use serde_json::{json, Value};
|
||
|
|
|
||
|
|
use crate::simulate::ask;
|
||
|
|
|
||
|
|
fn flag(argv: &[String], name: &str) -> Option<String> {
|
||
|
|
argv.iter().position(|a| a == name).and_then(|i| argv.get(i + 1)).cloned()
|
||
|
|
}
|
||
|
|
|
||
|
|
fn load(path: &str) -> anyhow::Result<Value> {
|
||
|
|
Ok(serde_json::from_str(&std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?)?)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// LIX = words per sentence + 100 * long words / words.
|
||
|
|
pub fn lix(text: &str) -> f64 {
|
||
|
|
let words: Vec<&str> = text
|
||
|
|
.split(|c: char| !(c.is_alphanumeric() || c == '\'' || c == '-'))
|
||
|
|
.filter(|w| w.chars().any(char::is_alphabetic))
|
||
|
|
.collect();
|
||
|
|
if words.is_empty() {
|
||
|
|
return 0.0;
|
||
|
|
}
|
||
|
|
let sentences = text.split(['.', '!', '?', ':', '\n']).filter(|s| s.chars().any(char::is_alphabetic)).count().max(1);
|
||
|
|
let long = words.iter().filter(|w| w.chars().count() > 6).count();
|
||
|
|
words.len() as f64 / sentences as f64 + 100.0 * long as f64 / words.len() as f64
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn lix_band(score: f64) -> &'static str {
|
||
|
|
match score {
|
||
|
|
s if s < 30.0 => "very easy",
|
||
|
|
s if s < 40.0 => "easy",
|
||
|
|
s if s < 50.0 => "takes effort",
|
||
|
|
_ => "official prose",
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/// The page as the visitor meets it: the question, every alternative,
|
||
|
|
/// and the form behind the one that serves them.
|
||
|
|
fn page_text(step: &Value, gutted: bool) -> String {
|
||
|
|
let mut text = format!("{}\n", step["question"].as_str().unwrap_or_default());
|
||
|
|
for (i, option) in step["options"].as_array().into_iter().flatten().enumerate() {
|
||
|
|
if gutted {
|
||
|
|
text.push_str(&format!("\n- Option {}\n", i + 1));
|
||
|
|
} else {
|
||
|
|
text.push_str(&format!(
|
||
|
|
"\n- {}\n {}\n",
|
||
|
|
option["name"].as_str().unwrap_or_default(),
|
||
|
|
option["description"].as_str().unwrap_or_default()
|
||
|
|
));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
let fields: Vec<&str> = step["fields"].as_array().into_iter().flatten().filter_map(|f| f["label"].as_str()).collect();
|
||
|
|
if !gutted && !fields.is_empty() {
|
||
|
|
text.push_str(&format!(
|
||
|
|
"\nThe form under \"{}\" asks for: {}.\n",
|
||
|
|
step["alternative"].as_str().unwrap_or_default(),
|
||
|
|
fields.join(", ")
|
||
|
|
));
|
||
|
|
}
|
||
|
|
text
|
||
|
|
}
|
||
|
|
|
||
|
|
/// What one page may ask of one person before it should be split into
|
||
|
|
/// a further question, or the fields moved to a later stage of the
|
||
|
|
/// case where they are actually needed. Complexity belongs in the
|
||
|
|
/// number of steps and states, not in the weight of any one page.
|
||
|
|
const MAX_OPTIONS: usize = 5;
|
||
|
|
const MAX_REQUIRED: usize = 5;
|
||
|
|
|
||
|
|
/// (alternatives offered, required fields in the form that serves them)
|
||
|
|
fn page_load(step: &Value) -> (usize, usize) {
|
||
|
|
(
|
||
|
|
step["options"].as_array().map_or(0, Vec::len),
|
||
|
|
step["fields"].as_array().into_iter().flatten().filter(|f| f["optional"] != true).count(),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
const QUESTIONS: [(&str, &str); 3] = [
|
||
|
|
("which", "Is it clear from the page which option is meant for someone in your situation?"),
|
||
|
|
("asked", "Can you tell from the page what you will be asked to provide?"),
|
||
|
|
("next", "Does the page tell you what happens after you send it, or who will read it?"),
|
||
|
|
];
|
||
|
|
|
||
|
|
fn understanding(cmd: &str, who: &str, page: &str, seed: usize) -> anyhow::Result<[bool; 3]> {
|
||
|
|
let prompt = format!(
|
||
|
|
"You are this person: {who}\n\nThis is the whole of a web page you have just opened:\n---\n{page}---\n\nAnswer three questions about it as yourself. Say yes only when the page text itself gives you that; if you would be guessing, say no.\n{}\nReply as JSON: {{\"which\": true or false, \"asked\": true or false, \"next\": true or false}}",
|
||
|
|
QUESTIONS.iter().map(|(k, q)| format!("- {k}: {q}")).collect::<Vec<_>>().join("\n")
|
||
|
|
);
|
||
|
|
let reply = ask(cmd, &prompt, &seed.to_string())?;
|
||
|
|
Ok([reply["which"] == true, reply["asked"] == true, reply["next"] == true])
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn rate(argv: &[String]) -> anyhow::Result<()> {
|
||
|
|
let usage = "usage: portal-trainer rate --model <model.json> --out <dir> [--llm 'cmd']";
|
||
|
|
let model = load(&flag(argv, "--model").ok_or_else(|| anyhow!(usage))?)?;
|
||
|
|
let out = std::path::PathBuf::from(flag(argv, "--out").ok_or_else(|| anyhow!(usage))?);
|
||
|
|
let cmd = flag(argv, "--llm").unwrap_or_else(|| "claude -p".to_string());
|
||
|
|
std::fs::create_dir_all(&out)?;
|
||
|
|
|
||
|
|
let mut report = BTreeMap::new();
|
||
|
|
println!("{:<28} {:>6} {:<14} {:>8} {:>8} {:>8} {:>16} {:<22}", "need", "LIX", "", "which", "asked", "next", "gutted twin says", "page load");
|
||
|
|
for need in model["needs"].as_array().into_iter().flatten() {
|
||
|
|
let id = need["id"].as_str().unwrap_or_default();
|
||
|
|
let Some(step) = need["steps"].as_array().and_then(|s| s.first()) else { continue };
|
||
|
|
let personas: Vec<&Value> = model["personas"].as_array().into_iter().flatten().filter(|p| p["need"] == id).collect();
|
||
|
|
if personas.is_empty() {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
let page = page_text(step, false);
|
||
|
|
let readability = lix(&page);
|
||
|
|
let (mut yes, mut control) = ([0usize; 3], [0usize; 3]);
|
||
|
|
for (i, persona) in personas.iter().enumerate() {
|
||
|
|
let who = persona["text"].as_str().unwrap_or_default();
|
||
|
|
for (tally, gutted) in [(&mut yes, false), (&mut control, true)] {
|
||
|
|
let answers = understanding(&cmd, who, &page_text(step, gutted), i)?;
|
||
|
|
for (slot, answer) in tally.iter_mut().zip(answers) {
|
||
|
|
*slot += answer as usize;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
let n = personas.len();
|
||
|
|
let share = |k: usize| format!("{}/{n}", yes[k]);
|
||
|
|
// The twin has no headings and no descriptions: "which" must
|
||
|
|
// collapse on it, or the instrument is agreeing with anything.
|
||
|
|
let control_ok = control[0] * 2 < n.max(1);
|
||
|
|
let (options, required) = page_load(step);
|
||
|
|
let steps = need["steps"].as_array().map_or(1, Vec::len);
|
||
|
|
println!(
|
||
|
|
"{:<28} {:>6.0} {:<14} {:>8} {:>8} {:>8} {:>16} {:<22}",
|
||
|
|
id,
|
||
|
|
readability,
|
||
|
|
lix_band(readability),
|
||
|
|
share(0),
|
||
|
|
share(1),
|
||
|
|
share(2),
|
||
|
|
format!("{}/{n} {}", control[0], if control_ok { "ok" } else { "CONTROL FAILED" }),
|
||
|
|
format!(
|
||
|
|
"{options} options, {required} required{}",
|
||
|
|
if options > MAX_OPTIONS || required > MAX_REQUIRED { " - SPLIT" } else { "" }
|
||
|
|
),
|
||
|
|
);
|
||
|
|
report.insert(
|
||
|
|
id.to_string(),
|
||
|
|
json!({
|
||
|
|
"lix": readability, "personas": n, "steps": steps,
|
||
|
|
"options": options, "required_fields": required,
|
||
|
|
"overloaded": options > MAX_OPTIONS || required > MAX_REQUIRED,
|
||
|
|
"understanding": {"which": yes[0], "asked": yes[1], "next": yes[2]},
|
||
|
|
"control_which": control[0], "control_ok": control_ok,
|
||
|
|
}),
|
||
|
|
);
|
||
|
|
}
|
||
|
|
std::fs::write(out.join("rate.json"), serde_json::to_string_pretty(&report)?)?;
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn prefer(argv: &[String]) -> anyhow::Result<()> {
|
||
|
|
let usage = "usage: portal-trainer prefer --a <model-a.json> --b <model-b.json> [--llm 'cmd']";
|
||
|
|
let a = load(&flag(argv, "--a").ok_or_else(|| anyhow!(usage))?)?;
|
||
|
|
let b = load(&flag(argv, "--b").ok_or_else(|| anyhow!(usage))?)?;
|
||
|
|
let cmd = flag(argv, "--llm").unwrap_or_else(|| "claude -p".to_string());
|
||
|
|
|
||
|
|
println!("{:<28} {:>5} {:>5} {:>14} {:>5} {:>5} {:>14} {:>9}", "need", "A", "B", "no preference", "A", "B", "no preference", "LIX A->B");
|
||
|
|
println!("{:<28} {:^26} {:^26}", "", "friendlier", "clearer");
|
||
|
|
for need_a in a["needs"].as_array().into_iter().flatten() {
|
||
|
|
let id = need_a["id"].as_str().unwrap_or_default();
|
||
|
|
let Some(need_b) = b["needs"].as_array().into_iter().flatten().find(|n| n["id"] == id) else { continue };
|
||
|
|
let (Some(step_a), Some(step_b)) = (need_a["steps"].as_array().and_then(|s| s.first()), need_b["steps"].as_array().and_then(|s| s.first())) else { continue };
|
||
|
|
let (text_a, text_b) = (page_text(step_a, false), page_text(step_b, false));
|
||
|
|
if text_a == text_b {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
// [friendlier, clearer] x [A, B, none]
|
||
|
|
let mut wins = [[0usize; 3]; 2];
|
||
|
|
for (i, persona) in a["personas"].as_array().into_iter().flatten().filter(|p| p["need"] == id).enumerate() {
|
||
|
|
let who = persona["text"].as_str().unwrap_or_default();
|
||
|
|
let mut picks: [[&str; 2]; 2] = [["", ""], ["", ""]];
|
||
|
|
for (order, (first, second)) in [(&text_a, &text_b), (&text_b, &text_a)].into_iter().enumerate() {
|
||
|
|
let prompt = format!(
|
||
|
|
"You are this person: {who}\n\nTwo versions of the same web page. They lead to the same thing; only the wording differs.\n\n=== VERSION 1 ===\n{first}\n=== VERSION 2 ===\n{second}\n\nAs yourself: which version makes you feel more welcome and at ease, and which one makes it clearer what to do?\nReply as JSON: {{\"friendlier\": 1 or 2, \"clearer\": 1 or 2}}"
|
||
|
|
);
|
||
|
|
let reply = ask(&cmd, &prompt, &(i * 2 + order).to_string())?;
|
||
|
|
for (k, key) in ["friendlier", "clearer"].iter().enumerate() {
|
||
|
|
let first_won = reply[key] == 1;
|
||
|
|
// In the second order, version 1 is B.
|
||
|
|
picks[k][order] = if first_won == (order == 0) { "A" } else { "B" };
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for k in 0..2 {
|
||
|
|
let slot = match picks[k] {
|
||
|
|
["A", "A"] => 0,
|
||
|
|
["B", "B"] => 1,
|
||
|
|
_ => 2,
|
||
|
|
};
|
||
|
|
wins[k][slot] += 1;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
println!(
|
||
|
|
"{:<28} {:>5} {:>5} {:>14} {:>5} {:>5} {:>14} {:>9}",
|
||
|
|
id, wins[0][0], wins[0][1], wins[0][2], wins[1][0], wins[1][1], wins[1][2],
|
||
|
|
format!("{:.0}->{:.0}", lix(&text_a), lix(&text_b))
|
||
|
|
);
|
||
|
|
}
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
#[cfg(test)]
|
||
|
|
mod tests {
|
||
|
|
use super::*;
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn lix_separates_plain_words_from_official_prose() {
|
||
|
|
let plain = "Get someone to look at a building. We come and see it. Then we tell you what it takes.";
|
||
|
|
let official = "Submission of documentation regarding rehabilitation requirements necessitates comprehensive structural evaluation procedures.";
|
||
|
|
assert!(lix(plain) < 30.0, "{}", lix(plain));
|
||
|
|
assert!(lix(official) > 60.0, "{}", lix(official));
|
||
|
|
assert_eq!(lix_band(lix(plain)), "very easy");
|
||
|
|
assert_eq!(lix(""), 0.0);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn the_gutted_twin_keeps_the_question_and_loses_the_wording() {
|
||
|
|
let step = json!({
|
||
|
|
"question": "What brings you here?",
|
||
|
|
"options": [{"name": "Get a price", "description": "For owners."}, {"name": "Work with us", "description": "For crews."}],
|
||
|
|
"alternative": "Get a price",
|
||
|
|
"fields": [{"label": "Email"}, {"label": "The build"}]
|
||
|
|
});
|
||
|
|
let page = page_text(&step, false);
|
||
|
|
assert!(page.contains("Get a price") && page.contains("asks for: Email, The build"));
|
||
|
|
let twin = page_text(&step, true);
|
||
|
|
assert!(twin.contains("What brings you here?") && twin.contains("Option 2"));
|
||
|
|
assert!(!twin.contains("Get a price") && !twin.contains("Email"));
|
||
|
|
}
|
||
|
|
}
|