rate and prefer: understanding, readability, friendliness, page load
Readability is counted (LIX), not judged. Understanding is three yes-or-no questions per persona, with a gutted twin of every page as the control: an instrument that still says yes to "Option 1, Option 2" is not measuring the page. Friendliness is a preference between two versions shown in both orders; a pick that flips with the order counts as no preference. Page load (options offered, required fields in the step) flags a page that should be split into a further question or have fields moved to a later stage. First run on the generated contractor site, qwen3:8b as the visitor: control passed on every need; "what happens next" was 0/2 everywhere and 2/2 everywhere once the page said who reads it and when (LIX 28 to 30); an official-voice control reads LIX 63 against 27. Friendliness mostly came back "no preference" at two personas per need: directionally right (never preferred the cold version), too thin to rely on. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01L4jrCgLiKKHAEFuZUJjckH
This commit is contained in:
Bendik Aagaard Lynghaug
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
612745bc66
commit
b9f2acb7b8
+253
@@ -0,0 +1,253 @@
|
||||
//! Understanding, readability, friendliness: three scores to iterate
|
||||
//! wording against, each measured the way that keeps it from sticking
|
||||
//! at its maximum.
|
||||
//!
|
||||
//! - **Readability** is counted, not judged: LIX, sentence length plus
|
||||
//! the share of words over six letters. Below 30 reads very easily,
|
||||
//! 30-40 easily, 40-50 takes effort, above 50 is official prose.
|
||||
//! - **Understanding** is three yes-or-no questions put to each persona
|
||||
//! about the page in front of them. `rate` also puts them to a
|
||||
//! deliberately gutted twin of the page (headings replaced by
|
||||
//! "Option 1", descriptions removed). An instrument that still says
|
||||
//! yes to the twin is not measuring the page, and `rate` says so.
|
||||
//! - **Friendliness** is a preference between two versions, never an
|
||||
//! absolute mark: `prefer` shows each persona both versions in both
|
||||
//! orders; a pick that flips with the order counts as no preference.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use anyhow::{anyhow, Context};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::simulate::ask;
|
||||
|
||||
fn flag(argv: &[String], name: &str) -> Option<String> {
|
||||
argv.iter().position(|a| a == name).and_then(|i| argv.get(i + 1)).cloned()
|
||||
}
|
||||
|
||||
fn load(path: &str) -> anyhow::Result<Value> {
|
||||
Ok(serde_json::from_str(&std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?)?)
|
||||
}
|
||||
|
||||
/// LIX = words per sentence + 100 * long words / words.
|
||||
pub fn lix(text: &str) -> f64 {
|
||||
let words: Vec<&str> = text
|
||||
.split(|c: char| !(c.is_alphanumeric() || c == '\'' || c == '-'))
|
||||
.filter(|w| w.chars().any(char::is_alphabetic))
|
||||
.collect();
|
||||
if words.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let sentences = text.split(['.', '!', '?', ':', '\n']).filter(|s| s.chars().any(char::is_alphabetic)).count().max(1);
|
||||
let long = words.iter().filter(|w| w.chars().count() > 6).count();
|
||||
words.len() as f64 / sentences as f64 + 100.0 * long as f64 / words.len() as f64
|
||||
}
|
||||
|
||||
pub fn lix_band(score: f64) -> &'static str {
|
||||
match score {
|
||||
s if s < 30.0 => "very easy",
|
||||
s if s < 40.0 => "easy",
|
||||
s if s < 50.0 => "takes effort",
|
||||
_ => "official prose",
|
||||
}
|
||||
}
|
||||
|
||||
/// The page as the visitor meets it: the question, every alternative,
|
||||
/// and the form behind the one that serves them.
|
||||
fn page_text(step: &Value, gutted: bool) -> String {
|
||||
let mut text = format!("{}\n", step["question"].as_str().unwrap_or_default());
|
||||
for (i, option) in step["options"].as_array().into_iter().flatten().enumerate() {
|
||||
if gutted {
|
||||
text.push_str(&format!("\n- Option {}\n", i + 1));
|
||||
} else {
|
||||
text.push_str(&format!(
|
||||
"\n- {}\n {}\n",
|
||||
option["name"].as_str().unwrap_or_default(),
|
||||
option["description"].as_str().unwrap_or_default()
|
||||
));
|
||||
}
|
||||
}
|
||||
let fields: Vec<&str> = step["fields"].as_array().into_iter().flatten().filter_map(|f| f["label"].as_str()).collect();
|
||||
if !gutted && !fields.is_empty() {
|
||||
text.push_str(&format!(
|
||||
"\nThe form under \"{}\" asks for: {}.\n",
|
||||
step["alternative"].as_str().unwrap_or_default(),
|
||||
fields.join(", ")
|
||||
));
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
/// What one page may ask of one person before it should be split into
|
||||
/// a further question, or the fields moved to a later stage of the
|
||||
/// case where they are actually needed. Complexity belongs in the
|
||||
/// number of steps and states, not in the weight of any one page.
|
||||
const MAX_OPTIONS: usize = 5;
|
||||
const MAX_REQUIRED: usize = 5;
|
||||
|
||||
/// (alternatives offered, required fields in the form that serves them)
|
||||
fn page_load(step: &Value) -> (usize, usize) {
|
||||
(
|
||||
step["options"].as_array().map_or(0, Vec::len),
|
||||
step["fields"].as_array().into_iter().flatten().filter(|f| f["optional"] != true).count(),
|
||||
)
|
||||
}
|
||||
|
||||
const QUESTIONS: [(&str, &str); 3] = [
|
||||
("which", "Is it clear from the page which option is meant for someone in your situation?"),
|
||||
("asked", "Can you tell from the page what you will be asked to provide?"),
|
||||
("next", "Does the page tell you what happens after you send it, or who will read it?"),
|
||||
];
|
||||
|
||||
fn understanding(cmd: &str, who: &str, page: &str, seed: usize) -> anyhow::Result<[bool; 3]> {
|
||||
let prompt = format!(
|
||||
"You are this person: {who}\n\nThis is the whole of a web page you have just opened:\n---\n{page}---\n\nAnswer three questions about it as yourself. Say yes only when the page text itself gives you that; if you would be guessing, say no.\n{}\nReply as JSON: {{\"which\": true or false, \"asked\": true or false, \"next\": true or false}}",
|
||||
QUESTIONS.iter().map(|(k, q)| format!("- {k}: {q}")).collect::<Vec<_>>().join("\n")
|
||||
);
|
||||
let reply = ask(cmd, &prompt, &seed.to_string())?;
|
||||
Ok([reply["which"] == true, reply["asked"] == true, reply["next"] == true])
|
||||
}
|
||||
|
||||
pub fn rate(argv: &[String]) -> anyhow::Result<()> {
|
||||
let usage = "usage: portal-trainer rate --model <model.json> --out <dir> [--llm 'cmd']";
|
||||
let model = load(&flag(argv, "--model").ok_or_else(|| anyhow!(usage))?)?;
|
||||
let out = std::path::PathBuf::from(flag(argv, "--out").ok_or_else(|| anyhow!(usage))?);
|
||||
let cmd = flag(argv, "--llm").unwrap_or_else(|| "claude -p".to_string());
|
||||
std::fs::create_dir_all(&out)?;
|
||||
|
||||
let mut report = BTreeMap::new();
|
||||
println!("{:<28} {:>6} {:<14} {:>8} {:>8} {:>8} {:>16} {:<22}", "need", "LIX", "", "which", "asked", "next", "gutted twin says", "page load");
|
||||
for need in model["needs"].as_array().into_iter().flatten() {
|
||||
let id = need["id"].as_str().unwrap_or_default();
|
||||
let Some(step) = need["steps"].as_array().and_then(|s| s.first()) else { continue };
|
||||
let personas: Vec<&Value> = model["personas"].as_array().into_iter().flatten().filter(|p| p["need"] == id).collect();
|
||||
if personas.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let page = page_text(step, false);
|
||||
let readability = lix(&page);
|
||||
let (mut yes, mut control) = ([0usize; 3], [0usize; 3]);
|
||||
for (i, persona) in personas.iter().enumerate() {
|
||||
let who = persona["text"].as_str().unwrap_or_default();
|
||||
for (tally, gutted) in [(&mut yes, false), (&mut control, true)] {
|
||||
let answers = understanding(&cmd, who, &page_text(step, gutted), i)?;
|
||||
for (slot, answer) in tally.iter_mut().zip(answers) {
|
||||
*slot += answer as usize;
|
||||
}
|
||||
}
|
||||
}
|
||||
let n = personas.len();
|
||||
let share = |k: usize| format!("{}/{n}", yes[k]);
|
||||
// The twin has no headings and no descriptions: "which" must
|
||||
// collapse on it, or the instrument is agreeing with anything.
|
||||
let control_ok = control[0] * 2 < n.max(1);
|
||||
let (options, required) = page_load(step);
|
||||
let steps = need["steps"].as_array().map_or(1, Vec::len);
|
||||
println!(
|
||||
"{:<28} {:>6.0} {:<14} {:>8} {:>8} {:>8} {:>16} {:<22}",
|
||||
id,
|
||||
readability,
|
||||
lix_band(readability),
|
||||
share(0),
|
||||
share(1),
|
||||
share(2),
|
||||
format!("{}/{n} {}", control[0], if control_ok { "ok" } else { "CONTROL FAILED" }),
|
||||
format!(
|
||||
"{options} options, {required} required{}",
|
||||
if options > MAX_OPTIONS || required > MAX_REQUIRED { " - SPLIT" } else { "" }
|
||||
),
|
||||
);
|
||||
report.insert(
|
||||
id.to_string(),
|
||||
json!({
|
||||
"lix": readability, "personas": n, "steps": steps,
|
||||
"options": options, "required_fields": required,
|
||||
"overloaded": options > MAX_OPTIONS || required > MAX_REQUIRED,
|
||||
"understanding": {"which": yes[0], "asked": yes[1], "next": yes[2]},
|
||||
"control_which": control[0], "control_ok": control_ok,
|
||||
}),
|
||||
);
|
||||
}
|
||||
std::fs::write(out.join("rate.json"), serde_json::to_string_pretty(&report)?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn prefer(argv: &[String]) -> anyhow::Result<()> {
|
||||
let usage = "usage: portal-trainer prefer --a <model-a.json> --b <model-b.json> [--llm 'cmd']";
|
||||
let a = load(&flag(argv, "--a").ok_or_else(|| anyhow!(usage))?)?;
|
||||
let b = load(&flag(argv, "--b").ok_or_else(|| anyhow!(usage))?)?;
|
||||
let cmd = flag(argv, "--llm").unwrap_or_else(|| "claude -p".to_string());
|
||||
|
||||
println!("{:<28} {:>5} {:>5} {:>14} {:>5} {:>5} {:>14} {:>9}", "need", "A", "B", "no preference", "A", "B", "no preference", "LIX A->B");
|
||||
println!("{:<28} {:^26} {:^26}", "", "friendlier", "clearer");
|
||||
for need_a in a["needs"].as_array().into_iter().flatten() {
|
||||
let id = need_a["id"].as_str().unwrap_or_default();
|
||||
let Some(need_b) = b["needs"].as_array().into_iter().flatten().find(|n| n["id"] == id) else { continue };
|
||||
let (Some(step_a), Some(step_b)) = (need_a["steps"].as_array().and_then(|s| s.first()), need_b["steps"].as_array().and_then(|s| s.first())) else { continue };
|
||||
let (text_a, text_b) = (page_text(step_a, false), page_text(step_b, false));
|
||||
if text_a == text_b {
|
||||
continue;
|
||||
}
|
||||
// [friendlier, clearer] x [A, B, none]
|
||||
let mut wins = [[0usize; 3]; 2];
|
||||
for (i, persona) in a["personas"].as_array().into_iter().flatten().filter(|p| p["need"] == id).enumerate() {
|
||||
let who = persona["text"].as_str().unwrap_or_default();
|
||||
let mut picks: [[&str; 2]; 2] = [["", ""], ["", ""]];
|
||||
for (order, (first, second)) in [(&text_a, &text_b), (&text_b, &text_a)].into_iter().enumerate() {
|
||||
let prompt = format!(
|
||||
"You are this person: {who}\n\nTwo versions of the same web page. They lead to the same thing; only the wording differs.\n\n=== VERSION 1 ===\n{first}\n=== VERSION 2 ===\n{second}\n\nAs yourself: which version makes you feel more welcome and at ease, and which one makes it clearer what to do?\nReply as JSON: {{\"friendlier\": 1 or 2, \"clearer\": 1 or 2}}"
|
||||
);
|
||||
let reply = ask(&cmd, &prompt, &(i * 2 + order).to_string())?;
|
||||
for (k, key) in ["friendlier", "clearer"].iter().enumerate() {
|
||||
let first_won = reply[key] == 1;
|
||||
// In the second order, version 1 is B.
|
||||
picks[k][order] = if first_won == (order == 0) { "A" } else { "B" };
|
||||
}
|
||||
}
|
||||
for k in 0..2 {
|
||||
let slot = match picks[k] {
|
||||
["A", "A"] => 0,
|
||||
["B", "B"] => 1,
|
||||
_ => 2,
|
||||
};
|
||||
wins[k][slot] += 1;
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"{:<28} {:>5} {:>5} {:>14} {:>5} {:>5} {:>14} {:>9}",
|
||||
id, wins[0][0], wins[0][1], wins[0][2], wins[1][0], wins[1][1], wins[1][2],
|
||||
format!("{:.0}->{:.0}", lix(&text_a), lix(&text_b))
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn lix_separates_plain_words_from_official_prose() {
|
||||
let plain = "Get someone to look at a building. We come and see it. Then we tell you what it takes.";
|
||||
let official = "Submission of documentation regarding rehabilitation requirements necessitates comprehensive structural evaluation procedures.";
|
||||
assert!(lix(plain) < 30.0, "{}", lix(plain));
|
||||
assert!(lix(official) > 60.0, "{}", lix(official));
|
||||
assert_eq!(lix_band(lix(plain)), "very easy");
|
||||
assert_eq!(lix(""), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gutted_twin_keeps_the_question_and_loses_the_wording() {
|
||||
let step = json!({
|
||||
"question": "What brings you here?",
|
||||
"options": [{"name": "Get a price", "description": "For owners."}, {"name": "Work with us", "description": "For crews."}],
|
||||
"alternative": "Get a price",
|
||||
"fields": [{"label": "Email"}, {"label": "The build"}]
|
||||
});
|
||||
let page = page_text(&step, false);
|
||||
assert!(page.contains("Get a price") && page.contains("asks for: Email, The build"));
|
||||
let twin = page_text(&step, true);
|
||||
assert!(twin.contains("What brings you here?") && twin.contains("Option 2"));
|
||||
assert!(!twin.contains("Get a price") && !twin.contains("Email"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user