Bendik Aagaard LynghaugandClaude Fable 5.1
9089503524
The inside of the business reads button labels and nothing else. For every desk and bucket with two or more buttons, a worker is told what just happened to a case, in the state's own words, and picks the button; scored right of total with what each miss was taken for. The vel's neighbour-help desk scores 2 of 4 and the notice board 1 of 2: the first numbers a desk has ever had. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
524 lines
27 KiB
Rust
524 lines
27 KiB
Rust
//! Understanding, readability, friendliness: three scores to iterate
|
|
//! wording against, each measured the way that keeps it from sticking
|
|
//! at its maximum.
|
|
//!
|
|
//! - **Readability** is counted, not judged: LIX, sentence length plus
|
|
//! the share of words over six letters, over the page's prose only -
|
|
//! the question, the descriptions and the followup - never headings
|
|
//! or field labels, which are not sentences and would drag the score
|
|
//! down to "very easy" whatever the prose says. Below 30 reads very
|
|
//! easily, 30-40 easily, 40-50 takes effort, above 50 is official
|
|
//! prose.
|
|
//! - **Understanding** is three yes-or-no questions put to each persona
|
|
//! about the page in front of them: which option is theirs, what the
|
|
//! form will ask, and what happens after sending. The last needs the
|
|
//! followup page, which `rate` reads from the content repo (`--repo`)
|
|
//! because the sim export stops at the form. `rate` also puts all
|
|
//! three to a deliberately gutted twin of the page (headings replaced
|
|
//! by "Option 1", descriptions, the form line and the followup
|
|
//! removed). An instrument that still says yes to the twin is not
|
|
//! measuring the page, and `rate` says so, per question. Every page
|
|
//! on a need's path is rated: behind a door, "which" is decided on
|
|
//! the door page and the form's page carries "asked" and "next".
|
|
//! - **Friendliness** is a preference between two versions, never an
|
|
//! absolute mark: `prefer` shows each persona both versions in both
|
|
//! orders; a pick that flips with the order counts as no preference.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::path::Path;
|
|
|
|
use anyhow::{anyhow, Context};
|
|
use serde_json::{json, Value};
|
|
|
|
use crate::card;
|
|
use crate::simulate::ask;
|
|
|
|
fn flag(argv: &[String], name: &str) -> Option<String> {
|
|
argv.iter().position(|a| a == name).and_then(|i| argv.get(i + 1)).cloned()
|
|
}
|
|
|
|
fn load(path: &str) -> anyhow::Result<Value> {
|
|
Ok(serde_json::from_str(&std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?)?)
|
|
}
|
|
|
|
/// LIX = words per sentence + 100 * long words / words.
|
|
pub fn lix(text: &str) -> f64 {
|
|
let words: Vec<&str> = text
|
|
.split(|c: char| !(c.is_alphanumeric() || c == '\'' || c == '-'))
|
|
.filter(|w| w.chars().any(char::is_alphabetic))
|
|
.collect();
|
|
if words.is_empty() {
|
|
return 0.0;
|
|
}
|
|
let sentences = text.split(['.', '!', '?', ':', '\n']).filter(|s| s.chars().any(char::is_alphabetic)).count().max(1);
|
|
let long = words.iter().filter(|w| w.chars().count() > 6).count();
|
|
words.len() as f64 / sentences as f64 + 100.0 * long as f64 / words.len() as f64
|
|
}
|
|
|
|
pub fn lix_band(score: f64) -> &'static str {
|
|
match score {
|
|
s if s < 30.0 => "very easy",
|
|
s if s < 40.0 => "easy",
|
|
s if s < 50.0 => "takes effort",
|
|
_ => "official prose",
|
|
}
|
|
}
|
|
|
|
/// The page as the visitor meets it: the question, every alternative,
|
|
/// the form behind the one that serves them, and the page they land on
|
|
/// after sending, when the repo is there to read it from.
|
|
fn page_text(step: &Value, page: Option<&Value>, followup: Option<&Value>, gutted: bool) -> String {
|
|
let mut text = match (gutted, page) {
|
|
(false, Some(p)) => card::page_head(p),
|
|
_ => format!("{}\n", step["question"].as_str().unwrap_or_default()),
|
|
};
|
|
// The cards that lead nowhere (the opener, the look around) are
|
|
// read before the options; the gutted twin loses them too.
|
|
if let Some(context) = step["context"].as_str().filter(|c| !c.trim().is_empty() && !gutted) {
|
|
text.push_str(&format!("\n{context}\n"));
|
|
}
|
|
for (i, option) in step["options"].as_array().into_iter().flatten().enumerate() {
|
|
if gutted {
|
|
text.push_str(&format!("\n- Option {}\n", i + 1));
|
|
continue;
|
|
}
|
|
// The whole card when the repo is there to read it from;
|
|
// heading and description otherwise.
|
|
let card = page
|
|
.and_then(|p| p["alternatives"].as_array())
|
|
.and_then(|alts| alts.iter().find(|a| a["name"] == option["name"]))
|
|
.map(|a| card::alternative(a, true));
|
|
match card {
|
|
Some(c) => text.push_str(&format!("\n{c}")),
|
|
None => text.push_str(&format!(
|
|
"\n- {}\n {}\n",
|
|
option["name"].as_str().unwrap_or_default(),
|
|
option["description"].as_str().unwrap_or_default()
|
|
)),
|
|
}
|
|
}
|
|
if page.is_some() && !gutted {
|
|
// The cards carry their own "Asks for" lines.
|
|
return finish_page_text(text, followup);
|
|
}
|
|
// The twin has no form line at all: "Field 1, Field 2" still
|
|
// tells a reader how many things they will be asked, and a model
|
|
// answers "can you tell what you will be asked" with yes to that
|
|
// (measured: 2 of 2 on every need).
|
|
let fields: Vec<&str> = step["fields"].as_array().into_iter().flatten().filter_map(|f| f["label"].as_str()).collect();
|
|
if !gutted && !fields.is_empty() {
|
|
text.push_str(&format!(
|
|
"\nThe form under \"{}\" asks for: {}.\n",
|
|
step["alternative"].as_str().unwrap_or_default(),
|
|
fields.join(", ")
|
|
));
|
|
}
|
|
if gutted {
|
|
return text;
|
|
}
|
|
finish_page_text(text, followup)
|
|
}
|
|
|
|
fn finish_page_text(mut text: String, followup: Option<&Value>) -> String {
|
|
if let Some(next) = followup {
|
|
text.push_str(&format!("\nAfter you send it, the next page says:\n{}\n", next["name"].as_str().unwrap_or_default()));
|
|
if let Some(d) = next["description"].as_str().filter(|d| !d.trim().is_empty()) {
|
|
text.push_str(&format!("{}\n", d.trim()));
|
|
}
|
|
for alt in next["alternatives"].as_array().into_iter().flatten() {
|
|
let d = alt["description"].as_str().unwrap_or_default().trim();
|
|
text.push_str(&format!("- {}{}{}\n", alt["name"].as_str().unwrap_or_default(), if d.is_empty() { "" } else { ": " }, d));
|
|
}
|
|
}
|
|
text
|
|
}
|
|
|
|
/// The running text a reader has to parse: the question, each
|
|
/// description, and the followup. Headings and labels are left out.
|
|
fn prose_text(step: &Value, followup: Option<&Value>) -> String {
|
|
let mut prose = vec![step["question"].as_str().unwrap_or_default().to_string()];
|
|
prose.extend(step["options"].as_array().into_iter().flatten().filter_map(|o| o["description"].as_str()).map(str::to_string));
|
|
if let Some(next) = followup {
|
|
prose.extend(next["description"].as_str().map(str::to_string));
|
|
prose.extend(next["alternatives"].as_array().into_iter().flatten().filter_map(|a| a["description"].as_str()).map(str::to_string));
|
|
}
|
|
prose.into_iter().filter(|p| !p.trim().is_empty()).collect::<Vec<_>>().join("\n")
|
|
}
|
|
|
|
/// The page a visitor lands on after the step's alternative: read from
|
|
/// the content repo, since the sim export stops at the form. `None`
|
|
/// when there is no repo, no `action`, or the page is not there.
|
|
fn followup_page(repo: Option<&Path>, step: &Value) -> Option<Value> {
|
|
let repo = repo?;
|
|
let page = load_page(repo, step["page"].as_str()?)?;
|
|
let alternative = step["alternative"].as_str()?;
|
|
let action = page["alternatives"].as_array()?.iter().find(|a| a["name"] == alternative)?["action"].as_str()?;
|
|
load_page(repo, action)
|
|
}
|
|
|
|
/// Every desk's buttons per bucket, from the simulation model:
|
|
/// (page, group, bucket, [(from, to, label)]). A desk with one button
|
|
/// for a bucket has nothing to confuse and is left out.
|
|
fn desk_buttons(model: &Value) -> Vec<(String, String, String, Vec<(String, String, String)>)> {
|
|
let mut out = Vec::new();
|
|
for (bucket, spec) in model["buckets"].as_object().into_iter().flatten() {
|
|
for desk in spec["desks"].as_array().into_iter().flatten() {
|
|
let moves: Vec<(String, String, String)> = desk["moves"]
|
|
.as_array()
|
|
.into_iter()
|
|
.flatten()
|
|
.map(|m| (s(&m["from"]), s(&m["to"]), s(&m["label"])))
|
|
.filter(|(_, _, l)| !l.is_empty())
|
|
.collect();
|
|
if moves.len() >= 2 {
|
|
out.push((s(&desk["page"]), s(&desk["group"]), bucket.clone(), moves));
|
|
}
|
|
}
|
|
}
|
|
out.sort();
|
|
out
|
|
}
|
|
|
|
fn s(v: &Value) -> String {
|
|
v.as_str().unwrap_or_default().to_string()
|
|
}
|
|
|
|
/// A worker on the desk, told what just happened to a case, picks
|
|
/// the button for it. The outcome is put in plain words from the
|
|
/// state's own name, never from the label, and the buttons are listed
|
|
/// in a seeded shuffle.
|
|
fn button_pick(cmd: &str, group: &str, bucket: &str, from: &str, to: &str, labels: &[&str], seed: usize) -> anyhow::Result<String> {
|
|
let plain = |id: &str| id.replace(['_', '-'], " ");
|
|
let mut order: Vec<&str> = labels.to_vec();
|
|
// A fixed rotation per move: the right answer is not always first.
|
|
let by = seed % order.len().max(1);
|
|
order.rotate_left(by);
|
|
let listed: Vec<String> = order.iter().enumerate().map(|(i, l)| format!("{}. {l}", i + 1)).collect();
|
|
let prompt = format!(
|
|
"You work on the {} desk of a small organisation. A case in \"{}\" was \"{}\" and has just become \"{}\". The desk shows these buttons for it:\n\n{}\n\nWhich button records what just happened? Reply as JSON: {{\"choice\": <the button's number>}}",
|
|
plain(group),
|
|
plain(bucket),
|
|
plain(from),
|
|
plain(to),
|
|
listed.join("\n")
|
|
);
|
|
let reply = ask(cmd, &prompt, &seed.to_string())?;
|
|
let n = reply["choice"].as_u64().unwrap_or(0) as usize;
|
|
Ok(order.get(n.wrapping_sub(1)).map(|l| l.to_string()).unwrap_or_default())
|
|
}
|
|
|
|
/// Every `mail:` on a state in the repo's aggregates.yaml, as
|
|
/// (`bucket:state`, subject, body) with the `{placeholders}` taken
|
|
/// out, so they read as the person will read them.
|
|
pub(crate) fn mail_texts(repo: &Path) -> Option<Vec<(String, String, String)>> {
|
|
let raw = std::fs::read_to_string(repo.join("aggregates.yaml")).ok()?;
|
|
let doc: Value = serde_yaml::from_str(&raw).ok()?;
|
|
let strip = |s: &str| {
|
|
let mut out = String::new();
|
|
let mut skip = false;
|
|
for c in s.chars() {
|
|
match c {
|
|
'{' => skip = true,
|
|
'}' => skip = false,
|
|
_ if !skip => out.push(c),
|
|
_ => {}
|
|
}
|
|
}
|
|
out.split_whitespace().collect::<Vec<_>>().join(" ")
|
|
};
|
|
let mut out = Vec::new();
|
|
for aggregate in doc["aggregates"].as_array().into_iter().flatten() {
|
|
let bucket = aggregate["bucket"].as_str().unwrap_or_default();
|
|
for (state, spec) in aggregate["states"].as_object().into_iter().flatten() {
|
|
if let Some(mail) = spec.get("mail") {
|
|
out.push((
|
|
format!("{bucket}:{state}"),
|
|
strip(mail["subject"].as_str().unwrap_or_default()),
|
|
strip(mail["body"].as_str().unwrap_or_default()),
|
|
));
|
|
}
|
|
}
|
|
}
|
|
Some(out)
|
|
}
|
|
|
|
/// `/` is `questions/index.yaml`; `/a/b` is `questions/a/b.yaml` or
|
|
/// `questions/a/b/index.yaml`.
|
|
pub(crate) fn load_page(repo: &Path, route: &str) -> Option<Value> {
|
|
let rel = route.trim_matches('/');
|
|
let candidates = if rel.is_empty() {
|
|
vec![repo.join("questions/index.yaml")]
|
|
} else {
|
|
vec![repo.join(format!("questions/{rel}.yaml")), repo.join(format!("questions/{rel}/index.yaml"))]
|
|
};
|
|
let text = candidates.iter().find_map(|p| std::fs::read_to_string(p).ok())?;
|
|
serde_yaml::from_str(&text).ok()
|
|
}
|
|
|
|
/// What one page may ask of one person before it should be split into
|
|
/// a further question, or the fields moved to a later stage of the
|
|
/// case where they are actually needed. Complexity belongs in the
|
|
/// number of steps and states, not in the weight of any one page.
|
|
const MAX_OPTIONS: usize = 5;
|
|
const MAX_REQUIRED: usize = 5;
|
|
|
|
/// (alternatives offered, required fields in the form that serves them)
|
|
fn page_load(step: &Value) -> (usize, usize) {
|
|
(
|
|
step["options"].as_array().map_or(0, Vec::len),
|
|
step["fields"].as_array().into_iter().flatten().filter(|f| f["optional"] != true).count(),
|
|
)
|
|
}
|
|
|
|
const QUESTIONS: [(&str, &str); 4] = [
|
|
("which", "Is it clear from the page which option is meant for someone in your situation?"),
|
|
("where", "Do you know where you are on this site, who this option is for, and what pressing its button will do?"),
|
|
("asked", "Can you tell from the page what you will be asked to provide?"),
|
|
("next", "Does the page tell you what happens after you send it, or who will read it?"),
|
|
];
|
|
const N: usize = QUESTIONS.len();
|
|
|
|
fn understanding(cmd: &str, who: &str, page: &str, seed: usize) -> anyhow::Result<[bool; N]> {
|
|
let prompt = format!(
|
|
"You are this person: {who}\n\nThis is the whole of a web page you have just opened:\n---\n{page}---\n\nAnswer four questions about it as yourself. Say yes only when the page text itself gives you that; if you would be guessing, say no.\n{}\nReply as JSON: {{\"which\": true or false, \"where\": true or false, \"asked\": true or false, \"next\": true or false}}",
|
|
QUESTIONS.iter().map(|(k, q)| format!("- {k}: {q}")).collect::<Vec<_>>().join("\n")
|
|
);
|
|
let reply = ask(cmd, &prompt, &seed.to_string())?;
|
|
Ok([reply["which"] == true, reply["where"] == true, reply["asked"] == true, reply["next"] == true])
|
|
}
|
|
|
|
pub fn rate(argv: &[String]) -> anyhow::Result<()> {
|
|
let usage = "usage: portal-trainer rate --model <model.json> --out <dir> [--repo <content dir>] [--llm 'cmd']";
|
|
let model = load(&flag(argv, "--model").ok_or_else(|| anyhow!(usage))?)?;
|
|
let out = std::path::PathBuf::from(flag(argv, "--out").ok_or_else(|| anyhow!(usage))?);
|
|
let repo = flag(argv, "--repo").map(std::path::PathBuf::from);
|
|
let cmd = flag(argv, "--llm").unwrap_or_else(|| "claude -p".to_string());
|
|
std::fs::create_dir_all(&out)?;
|
|
if repo.is_none() {
|
|
eprintln!("rate: no --repo, so no followup page is shown and \"next\" is not measured");
|
|
}
|
|
|
|
let mut report = BTreeMap::new();
|
|
// The desks: the inside of the business reads button labels and
|
|
// nothing else, so each desk is scored on whether a worker told
|
|
// what has just happened to a case picks the right button among
|
|
// that desk's buttons for the bucket.
|
|
let desks = desk_buttons(&model);
|
|
if !desks.is_empty() {
|
|
println!("{:<20} {:<24} {:>8} {}", "desk", "bucket", "buttons", "mistaken for");
|
|
for (page, group, bucket, moves) in &desks {
|
|
let labels: Vec<&str> = moves.iter().map(|(_, _, l)| l.as_str()).collect();
|
|
let mut right = 0;
|
|
let mut mistaken = Vec::new();
|
|
for (i, (from, to, label)) in moves.iter().enumerate() {
|
|
let picked = button_pick(&cmd, group, bucket, from, to, &labels, i)?;
|
|
if picked == *label {
|
|
right += 1;
|
|
} else {
|
|
mistaken.push(format!("{to} as {picked:?}"));
|
|
}
|
|
}
|
|
println!("{:<20} {:<24} {:>3} of {:<2} {}", page, bucket, right, moves.len(), mistaken.join("; "));
|
|
report.insert(format!("desk {page} {bucket}"), json!({ "right": right, "of": moves.len(), "mistaken": mistaken }));
|
|
}
|
|
println!();
|
|
}
|
|
// The mails first, when the repo carries any: a person reads a
|
|
// mail alone, with no card around it, so its prose is rated on
|
|
// its own - LIX and length.
|
|
if let Some(mails) = repo.as_deref().and_then(mail_texts) {
|
|
if !mails.is_empty() {
|
|
println!("{:<40} {:>6} {:<14} {:>6}", "mail", "LIX", "", "words");
|
|
for (name, subject, body) in &mails {
|
|
let text = format!("{subject}. {body}");
|
|
let score = lix(&text);
|
|
println!("{:<40} {:>6.0} {:<14} {:>6}", name, score, lix_band(score), text.split_whitespace().count());
|
|
report.insert(format!("mail {name}"), json!({ "lix": score, "words": text.split_whitespace().count() }));
|
|
}
|
|
println!();
|
|
}
|
|
}
|
|
println!("{:<28} {:>6} {:<14} {:>8} {:>8} {:>8} {:>8} {:<22} {:<22}", "need", "LIX", "", "which", "where", "asked", "next", "gutted twin says", "page load");
|
|
for need in model["needs"].as_array().into_iter().flatten() {
|
|
let id = need["id"].as_str().unwrap_or_default();
|
|
let steps = need["steps"].as_array().cloned().unwrap_or_default();
|
|
let personas: Vec<&Value> = model["personas"].as_array().into_iter().flatten().filter(|p| p["need"] == id).collect();
|
|
if personas.is_empty() {
|
|
continue;
|
|
}
|
|
// Every page on the path is rated. The followup follows only
|
|
// the last page's alternative: on a door page, what the
|
|
// alternative leads to is the next page, not what happens
|
|
// after sending.
|
|
let mut pages = Vec::new();
|
|
for (i, step) in steps.iter().enumerate() {
|
|
let followup = if i + 1 == steps.len() { followup_page(repo.as_deref(), step) } else { None };
|
|
let page = repo.as_deref().and_then(|r| load_page(r, step["page"].as_str().unwrap_or("/")));
|
|
let row = if steps.len() > 1 { format!("{id} @{}", step["page"].as_str().unwrap_or_default()) } else { id.to_string() };
|
|
let readability = lix(&prose_text(step, followup.as_ref()));
|
|
let (mut yes, mut control) = ([0usize; N], [0usize; N]);
|
|
for (i, persona) in personas.iter().enumerate() {
|
|
let who = persona["text"].as_str().unwrap_or_default();
|
|
for (tally, gutted) in [(&mut yes, false), (&mut control, true)] {
|
|
let answers = understanding(&cmd, who, &page_text(step, page.as_ref(), followup.as_ref(), gutted), i)?;
|
|
for (slot, answer) in tally.iter_mut().zip(answers) {
|
|
*slot += answer as usize;
|
|
}
|
|
}
|
|
}
|
|
let n = personas.len();
|
|
let measured = |k: usize| k != 3 || followup.is_some();
|
|
let share = |k: usize| if measured(k) { format!("{}/{n}", yes[k]) } else { "-".to_string() };
|
|
// The twin has no headings, no descriptions, no cards and no
|
|
// followup: every question must collapse on it, or that
|
|
// question is agreeing with anything.
|
|
let control_ok: Vec<bool> = (0..N).map(|k| !measured(k) || control[k] * 2 < n.max(1)).collect();
|
|
let failed: Vec<&str> = QUESTIONS.iter().zip(&control_ok).filter(|(_, ok)| !**ok).map(|((k, _), _)| *k).collect();
|
|
let (options, required) = page_load(step);
|
|
println!(
|
|
"{:<28} {:>6.0} {:<14} {:>8} {:>8} {:>8} {:>8} {:<22} {:<22}",
|
|
row,
|
|
readability,
|
|
lix_band(readability),
|
|
share(0),
|
|
share(1),
|
|
share(2),
|
|
share(3),
|
|
format!(
|
|
"{}/{}/{}/{} {}",
|
|
control[0],
|
|
control[1],
|
|
control[2],
|
|
if measured(3) { control[3].to_string() } else { "-".to_string() },
|
|
if failed.is_empty() { "ok".to_string() } else { format!("FAILED: {}", failed.join(", ")) }
|
|
),
|
|
format!(
|
|
"{options} options, {required} required{}",
|
|
if options > MAX_OPTIONS || required > MAX_REQUIRED { " - SPLIT" } else { "" }
|
|
),
|
|
);
|
|
pages.push(json!({
|
|
"page": step["page"], "lix": readability, "personas": n,
|
|
"options": options, "required_fields": required,
|
|
"overloaded": options > MAX_OPTIONS || required > MAX_REQUIRED,
|
|
"followup_shown": followup.is_some(),
|
|
"understanding": {"which": yes[0], "where": yes[1], "asked": yes[2], "next": if measured(3) { json!(yes[3]) } else { Value::Null }},
|
|
"control": {"which": control[0], "where": control[1], "asked": control[2], "next": if measured(3) { json!(control[3]) } else { Value::Null }},
|
|
"control_ok": failed.is_empty(),
|
|
}));
|
|
}
|
|
report.insert(id.to_string(), json!({ "steps": pages.len(), "pages": pages }));
|
|
}
|
|
std::fs::write(out.join("rate.json"), serde_json::to_string_pretty(&report)?)?;
|
|
Ok(())
|
|
}
|
|
|
|
pub fn prefer(argv: &[String]) -> anyhow::Result<()> {
|
|
let usage = "usage: portal-trainer prefer --a <model-a.json> --b <model-b.json> [--llm 'cmd']";
|
|
let a = load(&flag(argv, "--a").ok_or_else(|| anyhow!(usage))?)?;
|
|
let b = load(&flag(argv, "--b").ok_or_else(|| anyhow!(usage))?)?;
|
|
let cmd = flag(argv, "--llm").unwrap_or_else(|| "claude -p".to_string());
|
|
|
|
println!("{:<28} {:>5} {:>5} {:>14} {:>5} {:>5} {:>14} {:>9}", "need", "A", "B", "no preference", "A", "B", "no preference", "LIX A->B");
|
|
println!("{:<28} {:^26} {:^26}", "", "friendlier", "clearer");
|
|
for need_a in a["needs"].as_array().into_iter().flatten() {
|
|
let id = need_a["id"].as_str().unwrap_or_default();
|
|
let Some(need_b) = b["needs"].as_array().into_iter().flatten().find(|n| n["id"] == id) else { continue };
|
|
let (Some(step_a), Some(step_b)) = (need_a["steps"].as_array().and_then(|s| s.first()), need_b["steps"].as_array().and_then(|s| s.first())) else { continue };
|
|
let (text_a, text_b) = (page_text(step_a, None, None, false), page_text(step_b, None, None, false));
|
|
if text_a == text_b {
|
|
continue;
|
|
}
|
|
// [friendlier, clearer] x [A, B, none]
|
|
let mut wins = [[0usize; 3]; 2];
|
|
for (i, persona) in a["personas"].as_array().into_iter().flatten().filter(|p| p["need"] == id).enumerate() {
|
|
let who = persona["text"].as_str().unwrap_or_default();
|
|
let mut picks: [[&str; 2]; 2] = [["", ""], ["", ""]];
|
|
for (order, (first, second)) in [(&text_a, &text_b), (&text_b, &text_a)].into_iter().enumerate() {
|
|
let prompt = format!(
|
|
"You are this person: {who}\n\nTwo versions of the same web page. They lead to the same thing; only the wording differs.\n\n=== VERSION 1 ===\n{first}\n=== VERSION 2 ===\n{second}\n\nAs yourself: which version makes you feel more welcome and at ease, and which one makes it clearer what to do?\nReply as JSON: {{\"friendlier\": 1 or 2, \"clearer\": 1 or 2}}"
|
|
);
|
|
let reply = ask(&cmd, &prompt, &(i * 2 + order).to_string())?;
|
|
for (k, key) in ["friendlier", "clearer"].iter().enumerate() {
|
|
let first_won = reply[key] == 1;
|
|
// In the second order, version 1 is B.
|
|
picks[k][order] = if first_won == (order == 0) { "A" } else { "B" };
|
|
}
|
|
}
|
|
for k in 0..2 {
|
|
let slot = match picks[k] {
|
|
["A", "A"] => 0,
|
|
["B", "B"] => 1,
|
|
_ => 2,
|
|
};
|
|
wins[k][slot] += 1;
|
|
}
|
|
}
|
|
println!(
|
|
"{:<28} {:>5} {:>5} {:>14} {:>5} {:>5} {:>14} {:>9}",
|
|
id, wins[0][0], wins[0][1], wins[0][2], wins[1][0], wins[1][1], wins[1][2],
|
|
format!("{:.0}->{:.0}", lix(&text_a), lix(&text_b))
|
|
);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn lix_separates_plain_words_from_official_prose() {
|
|
let plain = "Get someone to look at a building. We come and see it. Then we tell you what it takes.";
|
|
let official = "Submission of documentation regarding rehabilitation requirements necessitates comprehensive structural evaluation procedures.";
|
|
assert!(lix(plain) < 30.0, "{}", lix(plain));
|
|
assert!(lix(official) > 60.0, "{}", lix(official));
|
|
assert_eq!(lix_band(lix(plain)), "very easy");
|
|
assert_eq!(lix(""), 0.0);
|
|
}
|
|
|
|
#[test]
|
|
fn the_gutted_twin_keeps_the_question_and_loses_the_wording() {
|
|
let step = json!({
|
|
"question": "What brings you here?",
|
|
"options": [{"name": "Get a price", "description": "For owners."}, {"name": "Work with us", "description": "For crews."}],
|
|
"alternative": "Get a price",
|
|
"fields": [{"label": "Email"}, {"label": "The build"}]
|
|
});
|
|
let next = json!({"name": "What happens now?", "description": "A project manager reads it.", "alternatives": [{"name": "We call you within two days", "description": ""}]});
|
|
let page = page_text(&step, None, Some(&next), false);
|
|
assert!(page.contains("Get a price") && page.contains("asks for: Email, The build"));
|
|
assert!(page.contains("After you send it") && page.contains("A project manager reads it."));
|
|
// With the repo's page in hand, each option is its whole card.
|
|
let repo_page = json!({"name": "What brings you here?", "description": "", "alternatives": [
|
|
{"name": "Get a price", "description": "For owners.", "consequence": ["Send it"], "encouragements": ["We answer within two days."], "features": [{"name": "Your brief", "requirements": [{"name": "email", "label": "Email"}]}]},
|
|
{"name": "Work with us", "description": "For crews."}]});
|
|
let rich = page_text(&step, Some(&repo_page), Some(&next), false);
|
|
assert!(rich.contains("Button: Send it") && rich.contains("We answer within two days.") && rich.contains("Asks for: Email"));
|
|
assert!(!rich.contains("asks for: Email, The build"));
|
|
let twin = page_text(&step, Some(&repo_page), Some(&next), true);
|
|
assert!(!twin.contains("Send it") && !twin.contains("Your brief"));
|
|
let twin = page_text(&step, None, Some(&next), true);
|
|
assert!(twin.contains("What brings you here?") && twin.contains("Option 2"));
|
|
assert!(!twin.contains("asks for"));
|
|
assert!(!twin.contains("Get a price") && !twin.contains("Email") && !twin.contains("project manager"));
|
|
// Readability is measured on the sentences, not the headings.
|
|
let prose = prose_text(&step, Some(&next));
|
|
assert_eq!(prose, "What brings you here?\nFor owners.\nFor crews.\nA project manager reads it.");
|
|
}
|
|
|
|
#[test]
|
|
fn the_followup_page_is_read_from_the_repo_by_following_the_action() {
|
|
let dir = std::env::temp_dir().join(format!("portal-trainer-rate-{}", std::process::id()));
|
|
std::fs::create_dir_all(dir.join("questions/received")).unwrap();
|
|
std::fs::write(dir.join("questions/index.yaml"), "name: What?\nalternatives:\n - name: Get a price\n action: /received/price\n").unwrap();
|
|
std::fs::write(dir.join("questions/received/price.yaml"), "name: What happens now?\ndescription: We read it.\nalternatives:\n - name: Back\n").unwrap();
|
|
let step = json!({"page": "/", "alternative": "Get a price"});
|
|
let next = followup_page(Some(&dir), &step).unwrap();
|
|
assert_eq!(next["name"], "What happens now?");
|
|
assert!(followup_page(None, &step).is_none());
|
|
assert!(followup_page(Some(&dir), &json!({"page": "/", "alternative": "Nope"})).is_none());
|
|
std::fs::remove_dir_all(dir).ok();
|
|
}
|
|
}
|