One binary: the lint and the trainer are commands of portal
Test / test (pull_request) Successful in 1m7s
Test / test (pull_request) Successful in 1m7s
portal serve runs a site; portal lint is what iris was; portal plan, build, report and the rest are what portal-trainer was. One source at one version, so the three never disagree about what content means. Both histories are merged in (subtree). The release attaches the bare binary for a content repo's CI. The trainer's plans, intakes, engines and reference prompt come along; its runs stay in the old repo. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GT37Z1Xtfd9pUuQtMTg6Yt
This commit is contained in:
Bendik Aagaard Lynghaug
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
d0d1cbf23d
commit
96b3746643
@@ -37,6 +37,13 @@ pub mod seed;
|
||||
pub mod memberships;
|
||||
#[cfg(feature = "ssr")]
|
||||
pub mod report;
|
||||
/// The lint: `portal lint`, what a content repo's CI runs (once uhhm/iris).
|
||||
#[cfg(feature = "ssr")]
|
||||
pub mod lint;
|
||||
/// The trainer: `portal plan`, `portal build`, `portal report` and the
|
||||
/// rest (once uhhm/portal-trainer).
|
||||
#[cfg(feature = "ssr")]
|
||||
pub mod trainer;
|
||||
|
||||
pub mod maps;
|
||||
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
//! Headless schema-check binary - loads content and `aggregates.yaml`
|
||||
//! (from a Gitea repo URL or a local directory) and validates them
|
||||
//! exactly the way `content::watch_for_reload`/`main.rs`'s boot path
|
||||
//! do, with no NATS, OIDC, web server, or JetStream connection
|
||||
//! involved. Built once by portal's own deploy workflow and run
|
||||
//! directly by `questions`' own CI (same bare-metal runner/host,
|
||||
//! published to a stable path - no artifact download needed), rather
|
||||
//! than compiled there - keeps that repo's CI coupling to "run a
|
||||
//! static binary," not "build a Rust workspace."
|
||||
|
||||
use crate::lint::local::{load_aggregates_from_dir, load_from_dir, load_needs_from_dir, load_site_from_dir};
|
||||
use crate::lint::needs;
|
||||
use crate::{aggregates, content};
|
||||
|
||||
pub async fn run(argv: Vec<String>) -> anyhow::Result<()> {
|
||||
let mut args = argv.into_iter();
|
||||
let mut repo: Option<String> = None;
|
||||
let mut branch = "main".to_string();
|
||||
let mut subdir = "questions".to_string();
|
||||
let mut path: Option<String> = None;
|
||||
let mut tasks_out: Option<String> = None;
|
||||
let mut answers_in: Option<String> = None;
|
||||
let mut min_accuracy: f64 = 0.0;
|
||||
let mut skim = false;
|
||||
let mut sim_out: Option<String> = None;
|
||||
let mut show_access = false;
|
||||
|
||||
while let Some(arg) = args.next() {
|
||||
match arg.as_str() {
|
||||
"--repo" => repo = args.next(),
|
||||
"--branch" => branch = args.next().unwrap_or(branch),
|
||||
"--subdir" => subdir = args.next().unwrap_or(subdir),
|
||||
"--path" => path = args.next(),
|
||||
"--needs-tasks" => tasks_out = args.next(),
|
||||
"--skim" => skim = true,
|
||||
"--needs-sim" => sim_out = args.next(),
|
||||
"--access" => show_access = true,
|
||||
"--needs-score" => answers_in = args.next(),
|
||||
"--min-accuracy" => {
|
||||
min_accuracy = args.next().and_then(|v| v.parse().ok()).unwrap_or(min_accuracy)
|
||||
}
|
||||
other => {
|
||||
eprintln!("unknown argument: {other}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let (questions, aggregates_map, site, needs_raw) = if let Some(dir) = path {
|
||||
(
|
||||
load_from_dir(&dir)?,
|
||||
load_aggregates_from_dir(&dir)?,
|
||||
load_site_from_dir(&dir)?,
|
||||
load_needs_from_dir(&dir)?,
|
||||
)
|
||||
} else if let Some(repo_url) = repo {
|
||||
// The first storyline of each, as the checks read it; the
|
||||
// loaders have held every list of texts to the site's arity.
|
||||
let questions = content::load_questions_from_gitea(&repo_url, &branch, &subdir).await?.at(0).clone();
|
||||
let aggregates_map = content::load_aggregates_from_gitea(&repo_url, &branch).await?.at(0).clone();
|
||||
// load_site_from_gitea already validates; a missing file is
|
||||
// the default config, same as at portal boot.
|
||||
let site = content::load_site_from_gitea(&repo_url, &branch).await?.at(0).clone();
|
||||
// Optional: a fetch failure means the repo declares no needs.
|
||||
let needs_raw = crate::http::get_text(&format!("{}/needs.yaml?ref={branch}", content::gitea_raw_base(&repo_url)?), crate::http::GITEA_CAP).await.ok();
|
||||
(questions, aggregates_map, site, needs_raw)
|
||||
} else {
|
||||
eprintln!(
|
||||
"usage: iris check (--repo <gitea-url> [--branch main] [--subdir questions] | --path <local-dir>) [--needs-tasks <out.jsonl> [--skim]] [--needs-sim <model.json>] [--access] [--needs-score <answers.jsonl> [--min-accuracy 0.8]]"
|
||||
);
|
||||
std::process::exit(2);
|
||||
};
|
||||
|
||||
if let Err(e) = site.validate() {
|
||||
eprintln!("FAIL: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
let aggregates_map = content::with_builtin_aggregates(aggregates_map);
|
||||
match content::validate_questions(&questions, &aggregates_map) {
|
||||
Ok(()) => {
|
||||
// The runtime's own event desk: a repo announcing pages
|
||||
// (Question.event) should read portal_events somewhere, or
|
||||
// "post what happened" tasks pile up unseen. Warn, don't
|
||||
// fail - the runtime creates the records either way.
|
||||
if questions.values().any(|q| q.event.is_some())
|
||||
&& !content::bucket_is_read(&questions, content::EVENTS_BUCKET)
|
||||
{
|
||||
eprintln!(
|
||||
"WARN: pages carry `event:` but no kv resource reads bucket {:?} - add a desk so summaries get posted",
|
||||
content::EVENTS_BUCKET
|
||||
);
|
||||
}
|
||||
println!(
|
||||
"OK: {} question(s), {} aggregate(s) valid",
|
||||
questions.len(),
|
||||
aggregates_map.len()
|
||||
);
|
||||
// The access policy the site will compile from this content.
|
||||
// It must type-check against portal's schema: a rule the
|
||||
// schema cannot express would grant nothing at runtime.
|
||||
let policy = crate::access::Policy::from_content(&questions, &aggregates_map);
|
||||
if let Err(e) = policy.validate() {
|
||||
eprintln!("FAIL: access policy does not validate: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
println!("OK: access policy, {} rule(s), validates", policy.rules.len());
|
||||
if show_access {
|
||||
println!("\n{:<16} {:<44} {:<40} {}", "ACTION", "RESOURCE", "WHO", "WHEN");
|
||||
let mut rows = policy.rules.clone();
|
||||
rows.sort_by(|a, b| (&a.resource, &a.action, &a.who).cmp(&(&b.resource, &b.action, &b.who)));
|
||||
for r in rows {
|
||||
println!("{:<16} {:<44} {:<40} {}", r.action, r.resource, r.who, r.when);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
check_needs(
|
||||
needs_raw.as_deref(),
|
||||
&questions,
|
||||
&aggregates_map,
|
||||
tasks_out.as_deref(),
|
||||
skim,
|
||||
sim_out.as_deref(),
|
||||
answers_in.as_deref(),
|
||||
min_accuracy,
|
||||
)
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("FAIL: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The business-needs pass (see `crate::lint::needs`): structure always,
|
||||
/// then the wording tasks written out and/or an engine's answers
|
||||
/// scored, when asked. A repo without `needs.yaml` skips all of it.
|
||||
fn check_needs(
|
||||
raw: Option<&str>,
|
||||
questions: &std::collections::HashMap<String, content::Question>,
|
||||
aggregates_map: &std::collections::HashMap<String, aggregates::AggregateSchema>,
|
||||
tasks_out: Option<&str>,
|
||||
skim: bool,
|
||||
sim_out: Option<&str>,
|
||||
answers_in: Option<&str>,
|
||||
min_accuracy: f64,
|
||||
) -> anyhow::Result<()> {
|
||||
let Some(raw) = raw else {
|
||||
if tasks_out.is_some() || answers_in.is_some() {
|
||||
eprintln!("FAIL: --needs-tasks/--needs-score given, but the repo has no needs.yaml");
|
||||
std::process::exit(1);
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
let file = match needs::parse(raw) {
|
||||
Ok(file) => file,
|
||||
Err(e) => {
|
||||
eprintln!("FAIL: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
let findings = needs::check(&file, questions, aggregates_map);
|
||||
let mut failed = 0;
|
||||
for finding in &findings {
|
||||
let level = match finding.level {
|
||||
needs::Level::Fail => {
|
||||
failed += 1;
|
||||
"FAIL"
|
||||
}
|
||||
needs::Level::Warn => "WARN",
|
||||
};
|
||||
eprintln!("{level}: need {:?}: {}", finding.need, finding.message);
|
||||
}
|
||||
for need in &file.needs {
|
||||
if let Some(path) = needs::path_for(questions, need) {
|
||||
let steps: Vec<String> = path
|
||||
.iter()
|
||||
.map(|s| format!("{} [{}]", s.page, s.alternative))
|
||||
.collect();
|
||||
println!("need {:?}: {} -> {:?}", need.id, steps.join(" -> "), need.lands_in);
|
||||
}
|
||||
}
|
||||
if failed == 0 {
|
||||
let planned = file.needs.iter().filter(|n| n.planned).count();
|
||||
println!(
|
||||
"OK: {} need(s) structurally met, {} planned, {} persona(s)",
|
||||
file.needs.len() - planned,
|
||||
planned,
|
||||
file.personas.len()
|
||||
);
|
||||
}
|
||||
// The wording pass still runs for the needs that do have a path -
|
||||
// a closed flow elsewhere is no reason to stop grading the rest.
|
||||
|
||||
if let Some(out) = sim_out {
|
||||
let model = needs::sim_model(&file, questions, aggregates_map);
|
||||
std::fs::write(out, serde_json::to_string_pretty(&model)?)
|
||||
.map_err(|e| anyhow::anyhow!("writing {out}: {e}"))?;
|
||||
println!("wrote the simulation model to {out}");
|
||||
}
|
||||
let tasks = needs::tasks(&file, questions, skim);
|
||||
if let Some(out) = tasks_out {
|
||||
let mut lines = String::new();
|
||||
for task in &tasks {
|
||||
lines.push_str(&serde_json::to_string(task)?);
|
||||
lines.push('\n');
|
||||
}
|
||||
std::fs::write(out, lines).map_err(|e| anyhow::anyhow!("writing {out}: {e}"))?;
|
||||
println!("wrote {} wording task(s) to {out}", tasks.len());
|
||||
}
|
||||
if let Some(path) = answers_in {
|
||||
let raw = std::fs::read_to_string(path).map_err(|e| anyhow::anyhow!("reading {path}: {e}"))?;
|
||||
let mut answers = Vec::new();
|
||||
for (n, line) in raw.lines().enumerate().filter(|(_, l)| !l.trim().is_empty()) {
|
||||
answers.push(
|
||||
serde_json::from_str::<needs::Answer>(line)
|
||||
.map_err(|e| anyhow::anyhow!("{path}:{}: {e}", n + 1))?,
|
||||
);
|
||||
}
|
||||
let score = needs::score(&tasks, &answers, 0.6);
|
||||
for (id, picked, expected, confidence) in &score.misroutes {
|
||||
eprintln!(
|
||||
"MISROUTE: {id}: picked {picked:?}{}, the need is met by {expected:?}",
|
||||
confidence.map(|c| format!(" ({c:.2})")).unwrap_or_default()
|
||||
);
|
||||
}
|
||||
for (id, confidence) in &score.hesitant {
|
||||
eprintln!("HESITANT: {id}: right, at {confidence:.2}");
|
||||
}
|
||||
for id in &score.unanswered {
|
||||
eprintln!("UNANSWERED: {id}");
|
||||
}
|
||||
println!(
|
||||
"wording: {}/{} routed right ({:.0}%)",
|
||||
score.correct,
|
||||
score.total,
|
||||
score.accuracy() * 100.0
|
||||
);
|
||||
if score.accuracy() < min_accuracy {
|
||||
eprintln!("FAIL: below --min-accuracy {min_accuracy}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
eprintln!("FAIL: {failed} unmet business need finding(s)");
|
||||
std::process::exit(1);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
//! Loading a content repo from a local checkout - the offline
|
||||
//! counterparts to the Gitea loaders in `content`, for tools that lint
|
||||
//! or replay a branch that has not been pushed (question_lint,
|
||||
//! needs_replay). `dir` is always the pages directory (`questions/`);
|
||||
//! `site.yaml`, `aggregates.yaml` and `needs.yaml` sit one level up.
|
||||
|
||||
use crate::{aggregates, content};
|
||||
|
||||
/// The offline counterpart to `content::load_site_from_gitea` -
|
||||
/// `site.yaml` lives at the repo root like `aggregates.yaml`, and is
|
||||
/// just as optional locally: missing means default branding.
|
||||
pub fn load_site_from_dir(dir: &str) -> anyhow::Result<content::SiteConfig> {
|
||||
let site_path = std::path::Path::new(dir)
|
||||
.parent()
|
||||
.unwrap_or_else(|| std::path::Path::new("."))
|
||||
.join("site.yaml");
|
||||
if !site_path.exists() {
|
||||
return Ok(content::SiteConfig::default());
|
||||
}
|
||||
let raw = std::fs::read_to_string(&site_path)
|
||||
.map_err(|e| anyhow::anyhow!("reading {}: {e}", site_path.display()))?;
|
||||
// The first storyline, held to the arity (content::variants).
|
||||
let files = vec![("site.yaml".to_string(), raw)];
|
||||
content::arity(&files)?;
|
||||
serde_yaml::from_str(&content::storyline(&files, 0)[0].1).map_err(|e| anyhow::anyhow!("parsing {}: {e}", site_path.display()))
|
||||
}
|
||||
|
||||
/// The offline counterpart to `content::load_aggregates_from_gitea` -
|
||||
/// `aggregates.yaml` lives at the repo root, one level up from the
|
||||
/// pages directory `--path` names, so `dir`'s parent is where it's
|
||||
/// looked for. Missing entirely is not an error here (unlike
|
||||
/// `--repo` mode, where `aggregates.yaml` is required) - a local
|
||||
/// checkout being linted may not have one, and every declared
|
||||
/// transition still gets checked against whatever *is* found; an
|
||||
/// empty map just means nothing is checked.
|
||||
pub fn load_aggregates_from_dir(
|
||||
dir: &str,
|
||||
) -> anyhow::Result<std::collections::HashMap<String, aggregates::AggregateSchema>> {
|
||||
let aggregates_path = std::path::Path::new(dir)
|
||||
.parent()
|
||||
.unwrap_or_else(|| std::path::Path::new("."))
|
||||
.join("aggregates.yaml");
|
||||
if !aggregates_path.exists() {
|
||||
return Ok(std::collections::HashMap::new());
|
||||
}
|
||||
let raw = std::fs::read_to_string(&aggregates_path)
|
||||
.map_err(|e| anyhow::anyhow!("reading {}: {e}", aggregates_path.display()))?;
|
||||
// The first storyline, held to the arity (content::variants).
|
||||
let files = vec![("aggregates.yaml".to_string(), raw)];
|
||||
content::arity(&files)?;
|
||||
aggregates::parse_aggregates_yaml(&content::storyline(&files, 0)[0].1)
|
||||
}
|
||||
|
||||
/// The offline counterpart to `content::load_questions_from_gitea` -
|
||||
/// the same recursive tree walk and `content::build_questions`
|
||||
/// pipeline (derived ids, relative refs, sections, followup
|
||||
/// inference), just reading a local checkout instead of Gitea's API,
|
||||
/// for linting a branch that hasn't been pushed yet.
|
||||
pub fn load_from_dir(
|
||||
dir: &str,
|
||||
) -> anyhow::Result<std::collections::HashMap<String, content::Question>> {
|
||||
let base = std::path::Path::new(dir);
|
||||
let mut files = Vec::new();
|
||||
collect_yaml(base, base, &mut files)?;
|
||||
content::build_questions(&files)
|
||||
}
|
||||
|
||||
fn collect_yaml(
|
||||
base: &std::path::Path,
|
||||
dir: &std::path::Path,
|
||||
out: &mut Vec<(String, String)>,
|
||||
) -> anyhow::Result<()> {
|
||||
for entry in std::fs::read_dir(dir)? {
|
||||
let path = entry?.path();
|
||||
if path.is_dir() {
|
||||
collect_yaml(base, &path, out)?;
|
||||
} else if path.extension().and_then(|e| e.to_str()) == Some("yaml") {
|
||||
let rel = path
|
||||
.strip_prefix(base)
|
||||
.expect("walked paths sit under their base")
|
||||
.to_string_lossy()
|
||||
.replace('\\', "/");
|
||||
let raw = std::fs::read_to_string(&path)
|
||||
.map_err(|e| anyhow::anyhow!("reading {}: {e}", path.display()))?;
|
||||
out.push((rel, raw));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The offline counterpart to `check`'s needs.yaml fetch -
|
||||
/// `needs.yaml` sits at the repo root beside `aggregates.yaml`.
|
||||
pub fn load_needs_from_dir(dir: &str) -> anyhow::Result<Option<String>> {
|
||||
let needs_path = std::path::Path::new(dir)
|
||||
.parent()
|
||||
.unwrap_or_else(|| std::path::Path::new("."))
|
||||
.join("needs.yaml");
|
||||
if !needs_path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
std::fs::read_to_string(&needs_path)
|
||||
.map(Some)
|
||||
.map_err(|e| anyhow::anyhow!("reading {}: {e}", needs_path.display()))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
//! iris - the shield over a portal site. A content push is an incoming
|
||||
//! wormhole; nothing comes through until it checks out.
|
||||
//!
|
||||
//! `iris check` loads a content repo (a Gitea URL or a local checkout)
|
||||
//! exactly the way the running site does, validates it, and holds it to
|
||||
//! the business needs it declares in `needs.yaml` - a path within
|
||||
//! budget for every kind of visitor, a desk for every handling group,
|
||||
//! every finished state reachable, no record stranding. It also exports
|
||||
//! what a trainer needs to grade the wording: personas as typed choice
|
||||
//! tasks, and a resolved simulation model.
|
||||
//!
|
||||
//! `iris replay` runs simulated cases through the site's real aggregate
|
||||
//! engine on a JetStream and reports every bucket by state; with
|
||||
//! `--report-only` it takes the same scorecard from a live site's
|
||||
//! buckets.
|
||||
//!
|
||||
//! Every type iris reads is portal's own, pinned to the portal release
|
||||
//! it matches, so the lint and the site never disagree about what a
|
||||
//! page is.
|
||||
|
||||
mod check;
|
||||
mod local;
|
||||
mod needs;
|
||||
mod replay;
|
||||
|
||||
/// The lint, as `portal lint ...`: `argv` is everything after `portal`.
|
||||
pub async fn run(mut argv: Vec<String>) -> anyhow::Result<()> {
|
||||
// `iris --path questions` reads as `iris check --path questions`,
|
||||
// so the content repos' one-line CI keeps working as it was.
|
||||
match argv.first().map(String::as_str) {
|
||||
Some("check") => {
|
||||
argv.remove(0);
|
||||
check::run(argv).await
|
||||
}
|
||||
Some("replay") => {
|
||||
argv.remove(0);
|
||||
replay::run(argv).await
|
||||
}
|
||||
Some("--version") | Some("-V") => {
|
||||
println!("portal lint v{}", env!("CARGO_PKG_VERSION"));
|
||||
Ok(())
|
||||
}
|
||||
Some(flag) if flag.starts_with("--") => check::run(argv).await,
|
||||
_ => {
|
||||
eprintln!(
|
||||
"iris {v} - matches portal v{v}\n\nusage:\n iris check (--repo <gitea-url> [--branch main] [--subdir questions] | --path <dir>) [--needs-tasks <out.jsonl> [--skim]] [--needs-sim <model.json>] [--access] [--needs-score <answers.jsonl> [--min-accuracy 0.8]]\n iris replay --path <dir> --nats <url> (--cases <cases.jsonl> | --report-only)",
|
||||
v = env!("CARGO_PKG_VERSION")
|
||||
);
|
||||
std::process::exit(2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1036
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,105 @@
|
||||
//! Makes a simulation's cases real: every case becomes an aggregate in
|
||||
//! its bucket, created and moved by the same code the running site
|
||||
//! uses - `answers::store_answer` for the submission,
|
||||
//! `aggregates::transition` for each desk move, against a real
|
||||
//! JetStream with its event log and its refusal of undeclared moves.
|
||||
//! Then the buckets are read back by replaying each record's events,
|
||||
//! and reported per need against `done_when` and `success`.
|
||||
//!
|
||||
//! The report half runs on its own (`--report-only`) against any
|
||||
//! JetStream, which is how the same scorecard is taken from a live
|
||||
//! site's buckets.
|
||||
//!
|
||||
//! Point it at a throwaway NATS, never a site's own: it writes.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
use futures::StreamExt;
|
||||
use crate::events::store::{ensure_stream, load_events};
|
||||
use crate::lint::local::{load_aggregates_from_dir, load_from_dir, load_needs_from_dir};
|
||||
use crate::lint::needs;
|
||||
use crate::{aggregates, answers, content};
|
||||
|
||||
pub async fn run(argv: Vec<String>) -> anyhow::Result<()> {
|
||||
let value = |flag: &str| argv.iter().position(|a| a == flag).and_then(|i| argv.get(i + 1)).cloned();
|
||||
let usage = "usage: iris replay --path <questions dir> --nats <url> (--cases <cases.jsonl> | --report-only)";
|
||||
let dir = value("--path").ok_or_else(|| anyhow::anyhow!(usage))?;
|
||||
let nats_url = value("--nats").ok_or_else(|| anyhow::anyhow!(usage))?;
|
||||
let cases_path = value("--cases");
|
||||
if cases_path.is_none() && !argv.iter().any(|a| a == "--report-only") {
|
||||
anyhow::bail!(usage);
|
||||
}
|
||||
|
||||
let questions = load_from_dir(&dir)?;
|
||||
let schemas = content::with_builtin_aggregates(load_aggregates_from_dir(&dir)?);
|
||||
let file = needs::parse(&load_needs_from_dir(&dir)?.ok_or_else(|| anyhow::anyhow!("the repo has no needs.yaml"))?)?;
|
||||
let js = async_nats::jetstream::new(async_nats::connect(&nats_url).await?);
|
||||
// The same call the site makes at boot.
|
||||
ensure_stream(&js).await?;
|
||||
|
||||
if let Some(path) = cases_path {
|
||||
let raw = std::fs::read_to_string(&path).map_err(|e| anyhow::anyhow!("reading {path}: {e}"))?;
|
||||
let (mut created, mut moved, mut refused) = (0, 0, 0);
|
||||
for (n, line) in raw.lines().filter(|l| !l.trim().is_empty()).enumerate() {
|
||||
let case: serde_json::Value = serde_json::from_str(line)?;
|
||||
let Some(need) = file.needs.iter().find(|need| case["need"] == need.id.as_str()) else { continue };
|
||||
let Some(schema) = schemas.get(&need.lands_in) else { continue };
|
||||
// Only a case that got as far as submitting leaves a record.
|
||||
let states: Vec<&str> = case["states"].as_array().into_iter().flatten().filter_map(|s| s.as_str()).collect();
|
||||
if states.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let step = needs::path_for(&questions, need).and_then(|p| p.last().cloned());
|
||||
let (page, alternative) = step.map(|s| (s.page, s.alternative)).unwrap_or_default();
|
||||
let id = format!("sim-{}-{}-{n}", case["persona"].as_str().unwrap_or("x"), case["run"]);
|
||||
let now = chrono::Utc::now().timestamp_millis();
|
||||
answers::store_answer(&js, &schemas, &need.lands_in, id.clone(), &page, &alternative, &case["record"], now, 0).await?;
|
||||
created += 1;
|
||||
for target in &states[1..] {
|
||||
match aggregates::transition(&js, schema, &id, target, serde_json::json!({"by": "simulation"}), now).await {
|
||||
Ok(_) => moved += 1,
|
||||
Err(e) => {
|
||||
refused += 1;
|
||||
eprintln!("REFUSED by the engine: {id} -> {target}: {e:?}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("{created} aggregate(s) created, {moved} transition(s) accepted, {refused} refused by the engine\n");
|
||||
}
|
||||
|
||||
// Read every bucket back the way the engine knows it: by replaying
|
||||
// each record's own event log.
|
||||
let mut by_bucket: HashMap<String, BTreeMap<String, usize>> = HashMap::new();
|
||||
for bucket in file.needs.iter().map(|n| n.lands_in.clone()).collect::<std::collections::BTreeSet<_>>() {
|
||||
let (Some(schema), Ok(store)) = (schemas.get(&bucket), js.get_key_value(&bucket).await) else { continue };
|
||||
let mut keys = store.keys().await?;
|
||||
while let Some(Ok(id)) = keys.next().await {
|
||||
let events = load_events(&js, &bucket, &id).await?;
|
||||
if let Some(record) = aggregates::replay(schema, &id, &events) {
|
||||
*by_bucket.entry(bucket.clone()).or_default().entry(record.state).or_default() += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("{:<18} {:>8} {:>9} {:>9} {}", "bucket", "records", "finished", "success", "by state");
|
||||
let mut buckets: Vec<_> = by_bucket.iter().collect();
|
||||
buckets.sort();
|
||||
for (bucket, states) in buckets {
|
||||
let needs_here: Vec<&needs::Need> = file.needs.iter().filter(|n| &n.lands_in == bucket).collect();
|
||||
let is = |pick: fn(&needs::Need) -> &Vec<String>, state: &String| needs_here.iter().any(|n| pick(n).contains(state));
|
||||
let total: usize = states.values().sum();
|
||||
let finished: usize = states.iter().filter(|(s, _)| is(|n| &n.done_when, s)).map(|(_, c)| c).sum();
|
||||
let success: usize = states.iter().filter(|(s, _)| is(|n| &n.success, s)).map(|(_, c)| c).sum();
|
||||
let pct = |n: usize| if total == 0 { "-".to_string() } else { format!("{:.0}%", n as f64 * 100.0 / total as f64) };
|
||||
println!(
|
||||
"{:<18} {:>8} {:>9} {:>9} {}",
|
||||
bucket,
|
||||
total,
|
||||
pct(finished),
|
||||
pct(success),
|
||||
states.iter().map(|(s, c)| format!("{s} {c}")).collect::<Vec<_>>().join(", ")
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
+25
@@ -1,6 +1,31 @@
|
||||
/// One binary, three jobs: `portal` (or `portal serve`) runs a site;
|
||||
/// `portal lint` checks a content repo the way the site reads it;
|
||||
/// `portal plan`, `build`, `report` and the rest train one. The lint
|
||||
/// and the trainer are the same code as the server, at the same
|
||||
/// version, so they never disagree about what content means.
|
||||
#[cfg(feature = "ssr")]
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let argv: Vec<String> = std::env::args().skip(1).collect();
|
||||
match argv.first().map(String::as_str) {
|
||||
Some("lint") => return portal::lint::run(argv[1..].to_vec()).await,
|
||||
Some("plan" | "build" | "answer" | "rate" | "prefer" | "score" | "simulate" | "compare" | "report" | "generate") => {
|
||||
// The trainer runs models through shell commands and waits
|
||||
// on them; off the runtime's threads, as a CLI should.
|
||||
return tokio::task::spawn_blocking(move || portal::trainer::run(argv)).await?;
|
||||
}
|
||||
Some("--version" | "-V") => {
|
||||
println!("portal v{}", env!("CARGO_PKG_VERSION"));
|
||||
return Ok(());
|
||||
}
|
||||
Some("serve") | None => {}
|
||||
Some(other) => anyhow::bail!("portal: unknown command {other:?}; one of serve, lint, plan, build, answer, rate, prefer, score, simulate, compare, report, generate"),
|
||||
}
|
||||
serve().await
|
||||
}
|
||||
|
||||
#[cfg(feature = "ssr")]
|
||||
async fn serve() -> anyhow::Result<()> {
|
||||
use axum::{body::Body, extract::State, http::Request, response::IntoResponse, routing::{any, get, post}, Router};
|
||||
use leptos::prelude::*;
|
||||
use leptos_axum::{generate_route_list, LeptosRoutes};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,101 @@
|
||||
//! An alternative as the visitor meets it, in text.
|
||||
//!
|
||||
//! Portal renders a card: the heading, the description, a button, one
|
||||
//! or two encouragements, and for each section an icon, a title, a
|
||||
//! line of guidance and the fields with their labels and placeholders.
|
||||
//! A visitor is oriented by all of it, not the heading alone. The
|
||||
//! answering visitor and `rate` both read this rendering, so the
|
||||
//! scored page is the served page; the gutted twin keeps the card's
|
||||
//! shape and none of its words.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
pub fn s(v: &Value) -> &str {
|
||||
v.as_str().unwrap_or_default()
|
||||
}
|
||||
|
||||
/// One option on a page. `full` is the card as rendered; otherwise the
|
||||
/// heading alone, which is what a visitor who reads nothing else sees.
|
||||
pub fn alternative(alt: &Value, full: bool) -> String {
|
||||
let mut out = format!("- {}", s(&alt["name"]));
|
||||
if !full {
|
||||
out.push('\n');
|
||||
return out;
|
||||
}
|
||||
let d = s(&alt["description"]).trim();
|
||||
if !d.is_empty() {
|
||||
out.push_str(&format!(": {d}"));
|
||||
}
|
||||
out.push('\n');
|
||||
if let Some(button) = alt["consequence"].as_array().and_then(|c| c.first()).and_then(Value::as_str) {
|
||||
out.push_str(&format!(" Button: {button}\n"));
|
||||
}
|
||||
for e in alt["encouragements"].as_array().into_iter().flatten().filter_map(Value::as_str) {
|
||||
out.push_str(&format!(" {e}\n"));
|
||||
}
|
||||
for f in alt["features"].as_array().into_iter().flatten() {
|
||||
let (name, desc, icon) = (s(&f["name"]).trim(), s(&f["description"]).trim(), s(&f["icon"]).trim());
|
||||
let mut line = String::from(" ");
|
||||
if !icon.is_empty() {
|
||||
line.push_str(&format!("[icon {}] ", icon.rsplit(':').next().unwrap_or(icon)));
|
||||
}
|
||||
if !name.is_empty() {
|
||||
line.push_str(name);
|
||||
}
|
||||
if !desc.is_empty() {
|
||||
line.push_str(&format!("{}{desc}", if name.is_empty() { "" } else { ": " }));
|
||||
}
|
||||
if !line.trim().is_empty() {
|
||||
out.push_str(line.trim_end());
|
||||
out.push('\n');
|
||||
}
|
||||
let fields: Vec<String> = f["requirements"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|r| {
|
||||
let label = r["label"].as_str().unwrap_or_else(|| s(&r["name"]));
|
||||
let hint = s(&r["placeholder"]).trim();
|
||||
let opt = if r["optional"] == true { " (optional)" } else { "" };
|
||||
if hint.is_empty() { format!("{label}{opt}") } else { format!("{label}{opt} e.g. {hint}") }
|
||||
})
|
||||
.collect();
|
||||
if !fields.is_empty() {
|
||||
out.push_str(&format!(" Asks for: {}\n", fields.join("; ")));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The page around its options: what it asks and says of itself.
|
||||
pub fn page_head(page: &Value) -> String {
|
||||
let mut out = format!("{}\n", s(&page["name"]));
|
||||
let d = s(&page["description"]).trim();
|
||||
if !d.is_empty() {
|
||||
out.push_str(&format!("{d}\n"));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn a_card_reads_as_the_visitor_sees_it_and_a_heading_alone_is_a_heading_alone() {
|
||||
let alt = json!({
|
||||
"name": "Get a price", "description": "For owners.",
|
||||
"consequence": ["Send it"], "encouragements": ["We answer within two days."],
|
||||
"features": [{"name": "Your brief", "icon": "lucide:file-text", "description": "A few lines are enough.",
|
||||
"requirements": [{"name": "email", "label": "Email", "placeholder": "you@example.no"},
|
||||
{"name": "address", "label": "Address", "optional": true}]}]
|
||||
});
|
||||
let full = alternative(&alt, true);
|
||||
assert!(full.starts_with("- Get a price: For owners.\n"));
|
||||
assert!(full.contains("Button: Send it") && full.contains("We answer within two days."));
|
||||
assert!(full.contains("[icon file-text] Your brief: A few lines are enough."));
|
||||
assert!(full.contains("Asks for: Email e.g. you@example.no; Address (optional)"));
|
||||
assert_eq!(alternative(&alt, false), "- Get a price\n");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,593 @@
|
||||
//! Generates a portal content repo from a business intake.
|
||||
//!
|
||||
//! Three phases, each held to portal's `question_lint`:
|
||||
//!
|
||||
//! 1. **Needs.** The intake (what the business does, who comes to it,
|
||||
//! what it keeps track of and through which stages, who handles
|
||||
//! what) becomes `needs.yaml`. After this phase the file is frozen:
|
||||
//! it is the spec, and the generator may not weaken it to pass.
|
||||
//! 2. **Structure.** Pages, `aggregates.yaml` and desks are generated
|
||||
//! and revised against the lint's findings until it exits zero.
|
||||
//! 3. **Wording.** The personas are answered blind (headings only,
|
||||
//! then with descriptions) and misroutes are fed back as wording
|
||||
//! defects. Only pages may change.
|
||||
//!
|
||||
//! The LLM is an external command: prompt on stdin, text on stdout.
|
||||
//! `claude -p` works; so does anything else with that shape.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::io::Write as _;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
use anyhow::{anyhow, bail, Context};
|
||||
|
||||
pub mod build;
|
||||
mod card;
|
||||
pub mod plan;
|
||||
mod rate;
|
||||
mod report;
|
||||
mod score;
|
||||
mod simulate;
|
||||
|
||||
pub const REFERENCE: &str = include_str!("reference.md");
|
||||
|
||||
struct Opts {
|
||||
intake: PathBuf,
|
||||
out: PathBuf,
|
||||
lint: String,
|
||||
llm: String,
|
||||
rounds: usize,
|
||||
wording_rounds: usize,
|
||||
/// The model that plays the visitor. A generator grading its own
|
||||
/// wording agrees with itself, so this should be a different one.
|
||||
visitor: Option<String>,
|
||||
}
|
||||
|
||||
/// The trainer, as `portal <command> ...`: `argv` is everything after
|
||||
/// `portal`. `answer --tasks <file> [--llm <cmd>]` is the blind
|
||||
/// answerer alone, for grading an existing site with any backend;
|
||||
/// answers go to stdout, ready for `portal lint --needs-score`.
|
||||
pub fn run(argv: Vec<String>) -> anyhow::Result<()> {
|
||||
match argv.first().map(String::as_str) {
|
||||
Some("simulate") => return simulate::run(&argv),
|
||||
Some("compare") => return simulate::compare(&argv),
|
||||
Some("plan") => return plan::run(&argv),
|
||||
Some("build") => return build::run(&argv),
|
||||
Some("rate") => return rate::rate(&argv),
|
||||
Some("report") => return report::run(&argv),
|
||||
Some("prefer") => return rate::prefer(&argv),
|
||||
Some("score") => return score::run(&argv),
|
||||
_ => {}
|
||||
}
|
||||
if argv.first().map(String::as_str) == Some("answer") {
|
||||
let value = |flag: &str| {
|
||||
argv.iter()
|
||||
.position(|a| a == flag)
|
||||
.and_then(|i| argv.get(i + 1))
|
||||
.cloned()
|
||||
};
|
||||
let tasks = value("--tasks").ok_or_else(|| anyhow!("usage: portal-trainer answer --tasks <tasks.jsonl> [--repo <content dir>] [--llm 'cmd'] [--seed <n>]"))?;
|
||||
let cmd = value("--llm").unwrap_or_else(|| "claude -p".to_string());
|
||||
let repo = value("--repo").map(PathBuf::from);
|
||||
// --seed: the same tasks answered again with other shuffles and
|
||||
// sampling, so a score comes with its spread. 0 is the run
|
||||
// every earlier score used.
|
||||
let seed: u64 = value("--seed").and_then(|s| s.parse().ok()).unwrap_or(0);
|
||||
print!("{}", answer_blind(&cmd, &std::fs::read_to_string(&tasks).with_context(|| format!("reading {tasks}"))?, repo.as_deref(), seed)?);
|
||||
return Ok(());
|
||||
}
|
||||
let opts = parse_args()?;
|
||||
let intake = std::fs::read_to_string(&opts.intake)
|
||||
.with_context(|| format!("reading {}", opts.intake.display()))?;
|
||||
std::fs::create_dir_all(opts.out.join("questions"))?;
|
||||
std::fs::create_dir_all(opts.out.join("llm"))?;
|
||||
LLM_LOG.set(opts.out.join("llm")).ok();
|
||||
let mut log = Vec::new();
|
||||
|
||||
// Phase 1: the spec.
|
||||
let mut feedback = String::new();
|
||||
for round in 1..=opts.rounds {
|
||||
let prompt = format!(
|
||||
"{REFERENCE}\n\n# Task\n\nA business filled in this intake. Write ONLY `needs.yaml` for it: every need the intake implies, for the public and for each role inside the business, with states taken from how the business says things progress, and personas.\n\n## Intake\n\n{intake}\n{feedback}\n{FORMAT}"
|
||||
);
|
||||
let Ok(files) = parse_files(&llm(&opts.llm, &prompt)?) else {
|
||||
eprintln!("needs round {round}: the reply had no file blocks, asking again");
|
||||
feedback = "\n## Your last attempt\n\nIt contained no <<<FILE ...>>> blocks at all. Return needs.yaml in exactly that format.\n".to_string();
|
||||
continue;
|
||||
};
|
||||
let Some(needs) = files.get("needs.yaml") else {
|
||||
feedback = "\n## Your last attempt\n\nIt contained no needs.yaml.\n".to_string();
|
||||
continue;
|
||||
};
|
||||
std::fs::write(opts.out.join("needs.yaml"), needs)?;
|
||||
// A placeholder front door, so the lint gets as far as parsing
|
||||
// the spec; every need fails, a parse error is what we look for.
|
||||
std::fs::write(opts.out.join("questions/index.yaml"), "name: Placeholder?\n")?;
|
||||
let (_, report) = lint(&opts, &[])?;
|
||||
if let Some(line) = report.lines().find(|l| l.contains("parsing needs.yaml")) {
|
||||
eprintln!("needs round {round}: {line}");
|
||||
feedback = format!("\n## Your last attempt failed to parse\n\n{line}\n\n```yaml\n{needs}\n```\n");
|
||||
continue;
|
||||
}
|
||||
eprintln!("needs round {round}: needs.yaml parses, frozen");
|
||||
log.push(format!("needs: parsed in round {round}"));
|
||||
feedback.clear();
|
||||
break;
|
||||
}
|
||||
if !feedback.is_empty() {
|
||||
bail!("needs.yaml never parsed in {} round(s); the replies are in {}/llm/", opts.rounds, opts.out.display());
|
||||
}
|
||||
let needs = std::fs::read_to_string(opts.out.join("needs.yaml")).context("no needs.yaml was produced")?;
|
||||
|
||||
// Phase 2: structure. The page-writing model sees the needs and
|
||||
// never the personas: pages written with the test in view route
|
||||
// their own personas perfectly and strangers badly (measured:
|
||||
// 10/10 on its own personas, 8/10 headings-only on held-out ones).
|
||||
let needs_full = needs;
|
||||
let needs = needs_full
|
||||
.split("\npersonas:")
|
||||
.next()
|
||||
.unwrap_or(&needs_full)
|
||||
.to_string();
|
||||
let mut green = false;
|
||||
let mut report = String::new();
|
||||
for round in 1..=opts.rounds {
|
||||
let prompt = if round == 1 {
|
||||
format!(
|
||||
"{REFERENCE}\n\n# Task\n\nWrite the content repo that meets this frozen `needs.yaml`: `aggregates.yaml`, `site.yaml`, and every page under `questions/` - the front door, followup pages, role-gated pages, one desk per handling group. Do not return needs.yaml.\n\n## Intake\n\n{intake}\n\n## needs.yaml (frozen)\n\n```yaml\n{needs}\n```\n\n{FORMAT}"
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"{REFERENCE}\n\n# Task\n\nThe verifier rejected the repo below. Fix it. Return only the files you change, each complete. To remove a file, return it with the single line DELETE. needs.yaml is frozen; do not return it.\n\n## Verifier output\n\n```\n{report}\n```\n\n## needs.yaml (frozen)\n\n```yaml\n{needs}\n```\n\n## Current files\n\n{}\n\n{FORMAT}",
|
||||
dump(&opts.out)?
|
||||
)
|
||||
};
|
||||
match parse_files(&llm(&opts.llm, &prompt)?) {
|
||||
Ok(files) => apply(&opts.out, files)?,
|
||||
Err(_) => eprintln!("structure round {round}: the reply had no file blocks"),
|
||||
}
|
||||
let (ok, r) = lint(&opts, &[])?;
|
||||
report = r;
|
||||
let fails = report.lines().filter(|l| l.starts_with("FAIL")).count();
|
||||
eprintln!("structure round {round}: {}", if ok { "green".to_string() } else { format!("{fails} FAIL line(s)") });
|
||||
log.push(format!("structure round {round}: ok={ok} fails={fails}"));
|
||||
if ok {
|
||||
green = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !green {
|
||||
std::fs::write(opts.out.join("run.log"), format!("{}\n\n{report}", log.join("\n")))?;
|
||||
bail!("structure never went green in {} round(s); last verifier output is in run.log", opts.rounds);
|
||||
}
|
||||
|
||||
// Phase 3: wording.
|
||||
for round in 0..=opts.wording_rounds {
|
||||
let mut defects = String::new();
|
||||
let mut summary = Vec::new();
|
||||
for skim in [true, false] {
|
||||
let tasks_path = opts.out.join(if skim { "tasks-skim.jsonl" } else { "tasks.jsonl" });
|
||||
let mut args = vec!["--needs-tasks".to_string(), tasks_path.display().to_string()];
|
||||
if skim {
|
||||
args.push("--skim".to_string());
|
||||
}
|
||||
lint(&opts, &args)?;
|
||||
let answers = answer_blind(opts.visitor.as_deref().unwrap_or(&opts.llm), &std::fs::read_to_string(&tasks_path)?, Some(&opts.out), 0)?;
|
||||
let answers_path = opts.out.join(if skim { "answers-skim.jsonl" } else { "answers.jsonl" });
|
||||
std::fs::write(&answers_path, answers)?;
|
||||
let mut args = vec!["--needs-score".to_string(), answers_path.display().to_string()];
|
||||
if skim {
|
||||
args.push("--skim".to_string());
|
||||
}
|
||||
let (_, scored) = lint(&opts, &args)?;
|
||||
let mode = if skim { "headings only" } else { "headings and descriptions" };
|
||||
for line in scored.lines().filter(|l| l.starts_with("MISROUTE") || l.starts_with("HESITANT")) {
|
||||
defects.push_str(&format!("[{mode}] {line}\n"));
|
||||
}
|
||||
if let Some(line) = scored.lines().find(|l| l.starts_with("wording:")) {
|
||||
summary.push(format!("{mode}: {line}"));
|
||||
}
|
||||
}
|
||||
eprintln!("wording round {round}: {}", summary.join(" | "));
|
||||
log.push(format!("wording round {round}: {}", summary.join(" | ")));
|
||||
if defects.is_empty() || round == opts.wording_rounds {
|
||||
break;
|
||||
}
|
||||
let personas = needs_full.split("\npersonas:").nth(1).unwrap_or_default();
|
||||
let prompt = format!(
|
||||
"{REFERENCE}\n\n# Task\n\nThe structure is correct. Visitors are picking the wrong alternative. Each MISROUTE below is a persona who, reading the page, chose an alternative that does not meet their need; HESITANT means they chose right and nearly did not. Rewrite alternative headings first (a heading that needs its description has failed), descriptions second. Keep every `record_as`, `action`, field and state as it is. If you rename an alternative that needs.yaml names in `via`, you may not: keep that one. Return only the pages you change, each complete.\n\n## Defects\n\n```\n{defects}```\n\n## The personas\n\n```yaml\npersonas:{personas}\n```\n\n## Current files\n\n{}\n\n{FORMAT}",
|
||||
dump(&opts.out)?
|
||||
);
|
||||
let before = snapshot(&opts.out)?;
|
||||
apply(&opts.out, parse_files(&llm(&opts.llm, &prompt)?)?)?;
|
||||
let (ok, r) = lint(&opts, &[])?;
|
||||
if !ok {
|
||||
eprintln!("wording round {round}: the rewrite broke structure, reverted");
|
||||
log.push(format!("wording round {round}: rewrite reverted\n{r}"));
|
||||
restore(&opts.out, before)?;
|
||||
break;
|
||||
}
|
||||
}
|
||||
std::fs::write(opts.out.join("run.log"), log.join("\n") + "\n")?;
|
||||
println!("content repo written to {}", opts.out.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const FORMAT: &str = "# Output format\n\nReturn files and nothing else, each as:\n\n<<<FILE relative/path.yaml>>>\n...complete file content...\n<<<END>>>\n\nNo prose, no code fences.";
|
||||
|
||||
fn parse_args() -> anyhow::Result<Opts> {
|
||||
let mut args = std::env::args().skip(1);
|
||||
if args.next().as_deref() != Some("generate") {
|
||||
bail!("usage: portal-trainer generate --intake <file> --out <dir> [--lint question_lint] [--llm 'claude -p'] [--rounds 4] [--wording-rounds 2]");
|
||||
}
|
||||
let mut opts = Opts {
|
||||
intake: PathBuf::new(),
|
||||
out: PathBuf::new(),
|
||||
lint: "question_lint".to_string(),
|
||||
llm: "claude -p".to_string(),
|
||||
rounds: 4,
|
||||
wording_rounds: 2,
|
||||
visitor: None,
|
||||
};
|
||||
while let Some(arg) = args.next() {
|
||||
let mut value = || args.next().ok_or_else(|| anyhow!("{arg} needs a value"));
|
||||
match arg.as_str() {
|
||||
"--intake" => opts.intake = value()?.into(),
|
||||
"--out" => opts.out = value()?.into(),
|
||||
"--lint" => opts.lint = value()?,
|
||||
"--llm" => opts.llm = value()?,
|
||||
"--rounds" => opts.rounds = value()?.parse()?,
|
||||
"--wording-rounds" => opts.wording_rounds = value()?.parse()?,
|
||||
"--visitor-llm" => opts.visitor = Some(value()?),
|
||||
other => bail!("unknown argument {other}"),
|
||||
}
|
||||
}
|
||||
if opts.intake.as_os_str().is_empty() || opts.out.as_os_str().is_empty() {
|
||||
bail!("--intake and --out are required");
|
||||
}
|
||||
Ok(opts)
|
||||
}
|
||||
|
||||
/// Prompt on stdin, text on stdout.
|
||||
/// Where `generate` keeps every prompt and reply (`<out>/llm/`). A
|
||||
/// model that returns nothing usable has to be read, not guessed at.
|
||||
static LLM_LOG: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
|
||||
static LLM_CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
fn llm(cmd: &str, prompt: &str) -> anyhow::Result<String> {
|
||||
llm_env(cmd, prompt, &[])
|
||||
}
|
||||
|
||||
/// As `llm`, with hints for the backend in its environment.
|
||||
fn llm_env(cmd: &str, prompt: &str, env: &[(&str, &str)]) -> anyhow::Result<String> {
|
||||
let reply = llm_with(cmd, prompt, env);
|
||||
if let Some(dir) = LLM_LOG.get() {
|
||||
let n = LLM_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
|
||||
std::fs::write(dir.join(format!("{n:02}-prompt.md")), prompt).ok();
|
||||
let text = match &reply {
|
||||
Ok(text) => text.clone(),
|
||||
Err(e) => format!("ERROR: {e}"),
|
||||
};
|
||||
std::fs::write(dir.join(format!("{n:02}-reply.md")), text).ok();
|
||||
}
|
||||
reply
|
||||
}
|
||||
|
||||
/// As `llm`, with hints for the backend in its environment (`SEED`,
|
||||
/// `FORMAT=json`, `TEMPERATURE`). A backend is free to ignore them.
|
||||
pub(crate) fn llm_with(cmd: &str, prompt: &str, env: &[(&str, &str)]) -> anyhow::Result<String> {
|
||||
let mut child = Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(cmd)
|
||||
.envs(env.iter().copied())
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.with_context(|| format!("starting {cmd:?}"))?;
|
||||
let mut stdin = child.stdin.take().expect("piped");
|
||||
let prompt = prompt.to_string();
|
||||
// Written from a thread: a large prompt would otherwise deadlock
|
||||
// against a child that starts writing before it finishes reading.
|
||||
let writer = std::thread::spawn(move || stdin.write_all(prompt.as_bytes()));
|
||||
let output = child.wait_with_output()?;
|
||||
writer.join().ok();
|
||||
if !output.status.success() {
|
||||
bail!("{cmd:?} exited with {}", output.status);
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&output.stdout).into_owned())
|
||||
}
|
||||
|
||||
fn lint(opts: &Opts, extra: &[String]) -> anyhow::Result<(bool, String)> {
|
||||
let output = Command::new(&opts.lint)
|
||||
.arg("--path")
|
||||
.arg(opts.out.join("questions"))
|
||||
.args(extra)
|
||||
.output()
|
||||
.with_context(|| format!("running {}", opts.lint))?;
|
||||
let mut report = String::from_utf8_lossy(&output.stdout).into_owned();
|
||||
report.push_str(&String::from_utf8_lossy(&output.stderr));
|
||||
Ok((output.status.success(), report))
|
||||
}
|
||||
|
||||
/// `<<<FILE path>>> ... <<<END>>>` blocks. Anything outside a block is
|
||||
/// ignored, so a model that adds a sentence does no harm.
|
||||
fn parse_files(text: &str) -> anyhow::Result<BTreeMap<String, String>> {
|
||||
// Small models mangle the markers: `<<<END>>` with a bracket short,
|
||||
// a stray `</think>`, the next `<<<FILE` glued onto the same line.
|
||||
// So markers are found anywhere in a line, an END needs only its
|
||||
// opening, and a new FILE closes whatever was open.
|
||||
let mut files = BTreeMap::new();
|
||||
let mut current: Option<(String, String)> = None;
|
||||
let close = |current: &mut Option<(String, String)>, files: &mut BTreeMap<String, String>| {
|
||||
if let Some((path, body)) = current.take() {
|
||||
files.insert(path, body);
|
||||
}
|
||||
};
|
||||
for line in text.lines() {
|
||||
let mut rest = line;
|
||||
loop {
|
||||
let (file_at, end_at) = (rest.find("<<<FILE "), rest.find("<<<END"));
|
||||
match (file_at, end_at) {
|
||||
(Some(f), e) if e.is_none_or(|e| f < e) => {
|
||||
close(&mut current, &mut files);
|
||||
let after = &rest[f + "<<<FILE ".len()..];
|
||||
let stop = after.find(">>>").unwrap_or(after.len());
|
||||
current = Some((after[..stop].trim().to_string(), String::new()));
|
||||
rest = after.get(stop + 3..).unwrap_or("");
|
||||
}
|
||||
(_, Some(e)) => {
|
||||
close(&mut current, &mut files);
|
||||
rest = &rest[e + "<<<END".len()..];
|
||||
}
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
// A line that carried a marker carries no content.
|
||||
if std::ptr::eq(rest, line) {
|
||||
if let Some((_, body)) = current.as_mut() {
|
||||
body.push_str(line);
|
||||
body.push('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
close(&mut current, &mut files);
|
||||
if files.is_empty() {
|
||||
bail!("the model returned no <<<FILE ...>>> blocks");
|
||||
}
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Model output is written to disk, so the path is checked, not
|
||||
/// trusted: repo-relative, yaml, and only where content lives.
|
||||
fn allowed(path: &str) -> bool {
|
||||
let p = Path::new(path);
|
||||
let plain = p.components().all(|c| matches!(c, std::path::Component::Normal(_)));
|
||||
plain
|
||||
&& p.extension().and_then(|e| e.to_str()) == Some("yaml")
|
||||
&& (matches!(path, "aggregates.yaml" | "site.yaml") || path.starts_with("questions/"))
|
||||
}
|
||||
|
||||
fn apply(out: &Path, files: BTreeMap<String, String>) -> anyhow::Result<()> {
|
||||
for (path, body) in files {
|
||||
if !allowed(&path) {
|
||||
eprintln!("ignored {path:?}: outside what the generator may write");
|
||||
continue;
|
||||
}
|
||||
let target = out.join(&path);
|
||||
if body.trim() == "DELETE" {
|
||||
std::fs::remove_file(&target).ok();
|
||||
continue;
|
||||
}
|
||||
if let Some(parent) = target.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
std::fs::write(target, body)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn content_files(out: &Path) -> anyhow::Result<Vec<PathBuf>> {
|
||||
fn walk(dir: &Path, found: &mut Vec<PathBuf>) -> anyhow::Result<()> {
|
||||
for entry in std::fs::read_dir(dir)? {
|
||||
let path = entry?.path();
|
||||
if path.is_dir() {
|
||||
walk(&path, found)?;
|
||||
} else if path.extension().and_then(|e| e.to_str()) == Some("yaml") {
|
||||
found.push(path);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
let mut found = Vec::new();
|
||||
for name in ["aggregates.yaml", "site.yaml"] {
|
||||
if out.join(name).exists() {
|
||||
found.push(out.join(name));
|
||||
}
|
||||
}
|
||||
walk(&out.join("questions"), &mut found)?;
|
||||
found.sort();
|
||||
Ok(found)
|
||||
}
|
||||
|
||||
fn snapshot(out: &Path) -> anyhow::Result<Vec<(PathBuf, String)>> {
|
||||
content_files(out)?
|
||||
.into_iter()
|
||||
.map(|p| Ok((p.clone(), std::fs::read_to_string(p)?)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn restore(out: &Path, before: Vec<(PathBuf, String)>) -> anyhow::Result<()> {
|
||||
for path in content_files(out)? {
|
||||
std::fs::remove_file(path)?;
|
||||
}
|
||||
for (path, body) in before {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
std::fs::write(path, body)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn dump(out: &Path) -> anyhow::Result<String> {
|
||||
let mut text = String::new();
|
||||
for path in content_files(out)? {
|
||||
let rel = path.strip_prefix(out).unwrap_or(&path).display().to_string();
|
||||
text.push_str(&format!("<<<FILE {rel}>>>\n{}<<<END>>>\n\n", std::fs::read_to_string(&path)?));
|
||||
}
|
||||
Ok(text)
|
||||
}
|
||||
|
||||
/// The answering side never sees `expect` or which need a persona has.
|
||||
///
|
||||
/// One call per task, so no visitor sees another visitor's situation
|
||||
/// (a batch of eight personas over four headings is nearly a matching
|
||||
/// puzzle, and a model solves it as one). The options are shuffled per
|
||||
/// task, so a heading's place on the page cannot carry the answer, and
|
||||
/// the call is seeded from the task id, so a rerun on the same site is
|
||||
/// the same experiment.
|
||||
///
|
||||
/// With a content repo to read, the visitor sees each option as the
|
||||
/// card portal renders: button, encouragement, sections with icon and
|
||||
/// guidance, fields with placeholders. Headings-only tasks (`--skim`,
|
||||
/// where the task's text is the heading itself) stay headings only.
|
||||
pub(crate) fn answer_blind(cmd: &str, tasks: &str, repo: Option<&Path>, seed_offset: u64) -> anyhow::Result<String> {
|
||||
let mut answers = String::new();
|
||||
for line in tasks.lines().filter(|l| !l.trim().is_empty()) {
|
||||
let task: serde_json::Value = serde_json::from_str(line)?;
|
||||
let id = task["id"].as_str().unwrap_or_default();
|
||||
let body = task["state"]["body"].as_str().unwrap_or_default();
|
||||
let pick = &task["questions"]["pick"];
|
||||
let instructions = pick["instructions"].as_str().unwrap_or_default();
|
||||
let mut options: Vec<(&str, &str)> = pick["criteria"]
|
||||
.as_object()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|(k, v)| (k.as_str(), v.as_str().unwrap_or_default()))
|
||||
.collect();
|
||||
let skim = options.iter().all(|(name, text)| text.is_empty() || text == name);
|
||||
let page = if skim { None } else { repo.and_then(|r| rate::load_page(r, task["page"].as_str().unwrap_or("/"))) };
|
||||
let seed = fnv(id).wrapping_add(seed_offset.wrapping_mul(0x9E37_79B9_7F4A_7C15) as _);
|
||||
shuffle(&mut options, seed);
|
||||
// Options are numbered and the answer is the number: with a
|
||||
// whole card in view, a small model asked to copy a heading
|
||||
// copied the page's description instead (measured: six of
|
||||
// fourteen misroutes on one run were that), and no heading
|
||||
// matched. A number cannot be a description.
|
||||
let listed: Vec<String> = options
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (name, text))| {
|
||||
let card = page
|
||||
.as_ref()
|
||||
.and_then(|p| p["alternatives"].as_array())
|
||||
.and_then(|alts| alts.iter().find(|a| a["name"] == *name))
|
||||
.map(|a| card::alternative(a, true));
|
||||
let body = match card {
|
||||
Some(card) => card.trim_end().trim_start_matches("- ").to_string(),
|
||||
None if skim => name.to_string(),
|
||||
None => format!("{name}: {text}"),
|
||||
};
|
||||
format!("{}. {body}", i + 1)
|
||||
})
|
||||
.collect();
|
||||
let mut head = page.as_ref().map(card::page_head).unwrap_or_default();
|
||||
// What the page says before it offers anything (the opener, the
|
||||
// look around): read, not picked.
|
||||
if let Some(context) = task["state"]["page"].as_str().filter(|c| !c.trim().is_empty()) {
|
||||
head.push_str(&format!("The page first says:\n{context}\n\n"));
|
||||
}
|
||||
let prompt = format!(
|
||||
"You are this person: {body}\n\n{instructions}\n{head}The page offers these options:\n\n{}\n\nWhich one do you pick? Judge only from your own situation and the wording in front of you. Give your honest confidence between 0 and 1; if two options fit you about equally, confidence is low.\nReply as JSON: {{\"choice\": <the option's number>, \"confidence\": 0.00}}",
|
||||
listed.join("\n\n")
|
||||
);
|
||||
let reply = llm_env(cmd, &prompt, &[("FORMAT", "json"), ("SEED", &seed.to_string()), ("TEMPERATURE", "0.2")])?;
|
||||
let parsed = simulate::first_object(&reply).unwrap_or(serde_json::Value::Null);
|
||||
let names: Vec<&str> = options.iter().map(|(n, _)| *n).collect();
|
||||
let raw = match &parsed["choice"] {
|
||||
serde_json::Value::Number(n) => n.to_string(),
|
||||
other => other.as_str().unwrap_or_default().to_string(),
|
||||
};
|
||||
// A number, or a heading for a model that wrote one anyway.
|
||||
let choice = match raw.trim().trim_end_matches('.').parse::<usize>() {
|
||||
Ok(n) if (1..=names.len()).contains(&n) => names[n - 1].to_string(),
|
||||
_ => simulate::resolve_choice(&names, &raw),
|
||||
};
|
||||
let confidence = parsed["confidence"].as_f64().unwrap_or(0.0);
|
||||
answers.push_str(&serde_json::to_string(&serde_json::json!({ "id": id, "choice": choice, "confidence": confidence }))?);
|
||||
answers.push('\n');
|
||||
}
|
||||
Ok(answers)
|
||||
}
|
||||
|
||||
/// FNV-1a over the task id: a seed that is the same on every run.
|
||||
fn fnv(text: &str) -> u32 {
|
||||
text.bytes().fold(0x811c_9dc5u32, |h, b| (h ^ u32::from(b)).wrapping_mul(0x0100_0193))
|
||||
}
|
||||
|
||||
/// Fisher-Yates on a small xorshift, so the order is fixed by the seed.
|
||||
fn shuffle<T>(items: &mut [T], seed: u32) {
|
||||
let mut x = seed.max(1);
|
||||
for i in (1..items.len()).rev() {
|
||||
x ^= x << 13;
|
||||
x ^= x >> 17;
|
||||
x ^= x << 5;
|
||||
items.swap(i, (x as usize) % (i + 1));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn files_are_parsed_from_blocks_and_prose_is_ignored() {
|
||||
let files = parse_files("Sure!\n<<<FILE site.yaml>>>\ntitle: A\n<<<END>>>\nnoise\n<<<FILE questions/index.yaml>>>\nname: Q?\n<<<END>>>\n").unwrap();
|
||||
assert_eq!(files["site.yaml"], "title: A\n");
|
||||
assert_eq!(files.len(), 2);
|
||||
assert!(parse_files("no blocks here").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mangled_markers_from_a_small_model_still_separate_the_files() {
|
||||
// Verbatim shape of a nemotron reply: a short END, a stray
|
||||
// think tag, the next FILE glued on, and no END at the very end.
|
||||
let files = parse_files(
|
||||
"<<<FILE questions/desk/index.yaml>>>\nname: Desk?\n<<<END>></think><<<FILE questions/index.yaml>>>\nname: Hva trenger du?\n<<<FILE site.yaml>>>\ntitle: A\n",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(files["questions/desk/index.yaml"], "name: Desk?\n");
|
||||
assert_eq!(files["questions/index.yaml"], "name: Hva trenger du?\n");
|
||||
assert_eq!(files["site.yaml"], "title: A\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_task_is_shuffled_the_same_way_every_time_and_differently_from_the_next() {
|
||||
let mut a = ["w", "x", "y", "z"];
|
||||
let mut b = a;
|
||||
shuffle(&mut a, fnv("scaffolder@/"));
|
||||
shuffle(&mut b, fnv("scaffolder@/"));
|
||||
assert_eq!(a, b);
|
||||
let mut c = ["w", "x", "y", "z"];
|
||||
shuffle(&mut c, fnv("apprentice@/"));
|
||||
assert_ne!(a, c);
|
||||
let mut sorted = a;
|
||||
sorted.sort();
|
||||
assert_eq!(sorted, ["w", "x", "y", "z"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_content_paths_may_be_written() {
|
||||
assert!(allowed("site.yaml"));
|
||||
assert!(allowed("aggregates.yaml"));
|
||||
assert!(allowed("questions/desk/_section.yaml"));
|
||||
// The spec is frozen after phase one.
|
||||
assert!(!allowed("needs.yaml"));
|
||||
assert!(!allowed("../outside.yaml"));
|
||||
assert!(!allowed("/etc/passwd.yaml"));
|
||||
assert!(!allowed("questions/run.sh"));
|
||||
assert!(!allowed(".gitea/workflows/deploy.yaml"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,495 @@
|
||||
//! From an intake, in plain words, to a `plan.yaml` a person can check.
|
||||
//!
|
||||
//! This is the one step where a model decides structure: which needs
|
||||
//! the intake implies, the stages each goes through, who handles it,
|
||||
//! what the form collects and how much that person will put up with.
|
||||
//! Everything after it is deterministic (`build`), so this is where a
|
||||
//! misreading of the business has to be caught, and it is caught by a
|
||||
//! person reading the plan, not by a verifier. The model's YAML is
|
||||
//! parsed and validated exactly as `build` would, and the errors go
|
||||
//! back to it for another round; what passes is written verbatim, so
|
||||
//! what the person reads is what the model wrote.
|
||||
//!
|
||||
//! Personas are never asked of this model: a persona written by the
|
||||
//! same model that phrased the needs tests nothing. They come from a
|
||||
//! person or a sector file, attached with `--personas`.
|
||||
|
||||
use anyhow::{anyhow, bail, Context};
|
||||
|
||||
use crate::trainer::build::Plan;
|
||||
use crate::trainer::simulate::first_object;
|
||||
|
||||
const REFERENCE: &str = r#"# plan.yaml
|
||||
|
||||
A plan is a needs spec that also says what each form collects. A
|
||||
builder turns it into a working site with no further decisions, so
|
||||
every structural fact about the business has to be in it.
|
||||
|
||||
```yaml
|
||||
business: <name>
|
||||
language: en # the intake's language code
|
||||
contact: { name: <who answers for the site>, email: <their email> }
|
||||
about: >
|
||||
What the business does, in its own words.
|
||||
mail: { from: "<name> <post@their-domain>", reply_to: <their inbox> }
|
||||
# who every mail the site sends comes from; required as soon
|
||||
# as any need has `mails`
|
||||
|
||||
does: # the front door's first card: what the business does, one
|
||||
- id: kebab-case # or two features, one role each, no button. Say it for the
|
||||
for: <need id> # role the business exists for and the next one it serves.
|
||||
text: what it does for them, in the intake's own words
|
||||
|
||||
shows: # what the site shows as it is: a live figure, the latest
|
||||
- id: kebab-case # releases, what has been built; `resource` is portal's block
|
||||
text: one line on what it is
|
||||
resource: { public: true, source: { kind: url, url: https://... } }
|
||||
|
||||
doors: # a first question before the forms, only when the offers
|
||||
- id: kebab-case # are more than one page holds (measured: four doors beat
|
||||
who: who comes through it # ten offers on one page; six offers on one page beat doors)
|
||||
|
||||
groups: # groups with a label the public may see: the committees
|
||||
- { id: traffic, label: Traffic, invited_by: [board] } # a person can ask to join (optional; label defaults to the id)
|
||||
invites_by: [board] # groups whose desk may invite people into every desk
|
||||
|
||||
needs:
|
||||
- id: kebab-case # ASCII only
|
||||
who: the kind of person, in plain words, in the intake's language
|
||||
wants: what they want from the business
|
||||
door: <door id> # the door this public need sits behind; omit to put it on the front door
|
||||
enters_as: <group> # ONLY for a role inside the business; omit for the public
|
||||
lands_in: <bucket> # snake_case, ASCII
|
||||
handled_by: <group> # the role that must see and move the record
|
||||
stages: [open, <stage>, ..., <end>, <end>]
|
||||
# EVERY stage the business said a case goes through, first to
|
||||
# last, read off their own sentences one verb at a time: "they
|
||||
# come in, we go and look, we send a quote, we win or lose it"
|
||||
# is [open, surveyed, quoted, won, lost]. A pipeline is not an
|
||||
# inbox: "answered" is rarely the real outcome.
|
||||
done_when: [<end>, <end>] # every stage that counts as finished, good or bad
|
||||
success: [<end>] # the finished stages that are a good outcome for the business
|
||||
skippable: [<stage>] # stages a case may pass over (papers already complete)
|
||||
requires: <one sentence> # what the person must ALREADY have or be for this to work:
|
||||
# a membership, a reference number, the link from a mail, an
|
||||
# account. Stated on the card above the form. Omit when there
|
||||
# is none; never invent one
|
||||
turned_down_from: [open, <stage>]
|
||||
# the unfinished stages a case can still be turned down from;
|
||||
# omit for every one. An applicant is declined before they
|
||||
# start, not after - after, they depart
|
||||
reached_from: { <stage>: [<stage>, ...] }
|
||||
# where a stage is entered from, when the line does not say it.
|
||||
# Two things need it and nothing else does: a case SENT BACK to
|
||||
# a stage it was already past ("papers missing" again, after the
|
||||
# hearing turned something up), and a GOOD ENDING partway along,
|
||||
# where a case finishes well without walking to the end ("we
|
||||
# changed our own decision", instead of passing it upwards).
|
||||
# A stage named here is entered from exactly those stages and
|
||||
# nowhere else, so it also says where a refusal may NOT come
|
||||
# from. Omit unless the business described one of those two
|
||||
moves_by: { <stage>: [<group>, ...] }
|
||||
# only where someone other than handled_by moves a case INTO a
|
||||
# stage: another group, or `submitter` (the person who sent the
|
||||
# form; the form must then collect a required `email`)
|
||||
watched_by: [<group>] # groups that see these cases and move nothing; `public` opens a
|
||||
# read-only page of the bucket to everyone (contact details hidden)
|
||||
mails: { open: submitter, <end>: submitter, <stage>: <address> }
|
||||
# who hears when a case enters a stage: `submitter` (the form
|
||||
# must then collect a required `email`) or one address. Tell the
|
||||
# person on arrival and on every decision that is about them.
|
||||
grants: { <stage>: <group> }
|
||||
# a stage that makes the person a member of a group when their
|
||||
# case arrives (an approval that IS membership); the form must
|
||||
# then collect a required `name` and a required `email`
|
||||
deadlines: { <stage>: { days: 30, to: <stage> } }
|
||||
# a stage a clock moves on from when a case sits in it that long
|
||||
# (a notice that expires, a hearing with no answer); only along
|
||||
# a move the stages already allow
|
||||
public_states: [<stage>] # with `public` in watched_by: the stages the public list shows
|
||||
# (a noticeboard shows what is up, not what was taken down)
|
||||
stages_text: { open: <words>, <stage>: <words> }
|
||||
# each stage in the intake's language, where the ASCII id
|
||||
# would read wrong to a person: "ikke_lost" is "ikke løst"
|
||||
events: { open: applied } # ONLY when rebuilding a live site: the event names its
|
||||
# aggregates.yaml already has, so its records keep replaying
|
||||
attended_by: > # ONLY for a stage no desk button moves: the automation that does
|
||||
what moves it, in a sentence
|
||||
max_fields: 4 # decide this BEFORE writing collects, from what the intake
|
||||
# says about the person; a budget that equals the form's size
|
||||
# is counted, not decided, and is reported as such
|
||||
# how many required fields THIS person will tolerate, from what
|
||||
# the intake says about them: someone on a phone with gloves
|
||||
# on gets 3, a first enquiry 4 to 6, a firm's paperwork 5 to 6
|
||||
collects: # the form; name is snake_case ASCII, type one of
|
||||
# text (default), email, tel, textarea, date, select, location
|
||||
# (a pin on a map, stored as lat/lon/zoom; optional: true,
|
||||
# since a map needs a browser that can draw one)
|
||||
- { name: name }
|
||||
- { name: phone, type: tel }
|
||||
- { name: problem, type: textarea }
|
||||
- { name: urgency, type: select, options: [This week, This month, No rush] }
|
||||
- { name: can_do, type: select, multiple: true, options: [Drive, Cook, Carry] } # any number
|
||||
- { name: committee, type: select, options_from: groups } # the labelled groups above
|
||||
- { name: address, optional: true, private: true } # never on a public page
|
||||
```
|
||||
|
||||
What a portal site can do - use what serves this business, leave the
|
||||
rest. Each is one line in the plan, and the builder does the wiring:
|
||||
|
||||
- **Show, not only ask.** A bucket with `watched_by: [public]` becomes
|
||||
a public list: its newest few records sit inline on the front door,
|
||||
linked to the whole list, contact details pruned. What neighbours
|
||||
reported and where it stands, which changes were approved, what is
|
||||
on the noticeboard: a visitor who sees the place working trusts it,
|
||||
and a person who reported something sees it being dealt with.
|
||||
`public_states` shows only the stages worth showing.
|
||||
- **Stay in dialogue.** `mails` tells the person at the stages that
|
||||
are about them, and a reply to that mail comes back to the case as
|
||||
a note. `moves_by: { <stage>: [submitter] }` gives them a move of
|
||||
their own through their link: confirm, withdraw, say it is solved.
|
||||
- **Let a decision do the work.** `grants` makes an approval into
|
||||
membership: the person gets an account and a place in the group the
|
||||
moment the case gets there; `deadlines` lets a stage end on its own
|
||||
(a notice that expires, a request nobody took up); `invites_by` lets
|
||||
a desk bring a person in without anyone touching an admin tool.
|
||||
- **Let nothing wait unseen.** `escalate: { open: { days: 7, to: styret } }`
|
||||
sends a case nobody has moved in a week up to another group's desk,
|
||||
where it sits with the same buttons until someone finishes it; the
|
||||
desk that had it keeps it too, and may send it up itself. A desk job,
|
||||
not a mail: the group sees it when they sit down.
|
||||
- **Let one record answer another.** A need `about` another need is
|
||||
one answer per record of it, from that record's own row on the
|
||||
public list: a neighbour saying "I can lend you that" to a notice, a
|
||||
bid to a request. `mails: { open: owner }` sends the answer straight
|
||||
to whoever posted the record, and the site keeps no queue - the
|
||||
poster's inbox is the queue. The site never becomes a go-between.
|
||||
- **Keep some things for members.** A need with `door` and `enters_as`
|
||||
is offered to everyone on the door and taken on the group's page
|
||||
behind sign-in, with name and address filled from the account: a
|
||||
seat on a committee for members, a discount for customers. The
|
||||
group is one a decision `grants`, so nobody has to be waved in.
|
||||
- **Tell it two ways and find out.** `tones` lists tones of voice, one
|
||||
storyline each: every word on the site is written once per tone, a
|
||||
visitor reads one storyline throughout, and the site counts what
|
||||
each brought in. Two is plenty; name them as a person would ("plain,
|
||||
like a good notice on the door"; "warm, as a neighbour would put
|
||||
it"). Leave it out to tell one story.
|
||||
- **Keep the people in portal.** `people: memberships` keeps every
|
||||
group in portal itself: a desk invite is a place held until the
|
||||
person signs in, a person is their phone number first, and any
|
||||
sign-in provider that says who someone is will do (Vipps, for a
|
||||
Norwegian association). Invite forms then ask for a phone number.
|
||||
Without it, Kanidm makes an account per invite.
|
||||
- **Answer for the pages as a group.** `contact: { name, email, group }`
|
||||
makes the contact a group's inbox rather than a person: no account
|
||||
is made for it, and every member of the group may suggest changes
|
||||
to the pages from the page itself.
|
||||
- **Make the choices real.** `groups` with labels, and a select with
|
||||
`options_from: groups`, so a form offers exactly the committees or
|
||||
teams that exist; `multiple: true` where a person can pick several.
|
||||
- **Place things.** A `location` field puts a pin on a map (where the
|
||||
hazard is, where the job is), when the site has maps.
|
||||
- **Open with the business itself.** `does` says what it does for the
|
||||
roles it exists for; `shows` puts its live content beside that.
|
||||
|
||||
Rules:
|
||||
- One need per (kind of person, thing they want). Work inside the
|
||||
business counts: a role recording or posting something is a need
|
||||
with `enters_as`. The intake's "what slows us down" usually names one.
|
||||
- Groups are the roles the intake names, as snake_case identifiers.
|
||||
- The front door opens with `does` (at most two features, one role
|
||||
each) and `shows`, then holds at most seven offers: doors plus needs
|
||||
with no door. A door with no need behind it is refused; a door on a
|
||||
need with `enters_as` is the way to the members' page. A role `does`
|
||||
speaks for must have a move
|
||||
of its own after sending (`moves_by` with `submitter`) or a public
|
||||
view, or it is a spectator.
|
||||
- Every need whose form collects an email should tell the person on
|
||||
arrival and on the decisions about them (`mails`); the plan then
|
||||
needs `mail: { from }`.
|
||||
- Every feature on a card explains itself: a list says what one row is
|
||||
and what its buttons do, a form says what to have ready. If an
|
||||
alternative only works for someone who already has something, say so
|
||||
in `requires` - a condition the reader meets only after filling the
|
||||
form in has already cost them the filling in.
|
||||
- Needs that land in the same bucket must have the same stages,
|
||||
done_when, skippable, moves_by, watched_by and handled_by.
|
||||
- Every required field is one the business will use to act on the
|
||||
record. If the intake gives no reason to need it, make it optional or
|
||||
leave it out. The required fields must not exceed max_fields.
|
||||
- Only `select` lists `options`, and it must list them.
|
||||
- Everything a person reads (who, wants, about, select options) is in
|
||||
the intake's language. Everything a machine reads (ids, buckets,
|
||||
groups, stages, field names) is plain ASCII: no æ, ø, å.
|
||||
- Write no personas.
|
||||
"#;
|
||||
|
||||
pub fn run(argv: &[String]) -> anyhow::Result<()> {
|
||||
let value = |flag: &str| {
|
||||
argv.iter()
|
||||
.position(|a| a == flag)
|
||||
.and_then(|i| argv.get(i + 1))
|
||||
.cloned()
|
||||
};
|
||||
let usage = "usage: portal-trainer plan --intake <file> --out <dir> [--llm 'cmd'] [--personas <yaml with personas:>] [--rounds 4]";
|
||||
let intake_path = value("--intake").ok_or_else(|| anyhow!(usage))?;
|
||||
let out = std::path::PathBuf::from(value("--out").ok_or_else(|| anyhow!(usage))?);
|
||||
let cmd = value("--llm").unwrap_or_else(|| "claude -p".to_string());
|
||||
let rounds: usize = value("--rounds")
|
||||
.map(|v| v.parse())
|
||||
.transpose()?
|
||||
.unwrap_or(4);
|
||||
let intake =
|
||||
std::fs::read_to_string(&intake_path).with_context(|| format!("reading {intake_path}"))?;
|
||||
let mut personas: Option<serde_yaml::Value> = match value("--personas") {
|
||||
Some(path) => {
|
||||
let text = std::fs::read_to_string(&path).with_context(|| format!("reading {path}"))?;
|
||||
let doc: serde_yaml::Value =
|
||||
serde_yaml::from_str(&text).with_context(|| format!("parsing {path}"))?;
|
||||
Some(doc.get("personas").cloned().ok_or_else(|| anyhow!("{path} has no `personas:`"))?)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
std::fs::create_dir_all(out.join("llm"))?;
|
||||
|
||||
let mut feedback = String::new();
|
||||
// A structural finding the builder would only warn about (a role
|
||||
// the opener speaks for with nothing to play) goes back to the
|
||||
// model once; a second time it is the reader's, not another round.
|
||||
let mut spectators_sent = false;
|
||||
for round in 1..=rounds {
|
||||
let prompt = format!(
|
||||
"{REFERENCE}\n\n# Task\n\nA business filled in this intake. Write its plan.yaml: every need the intake implies, for the public and for each role inside the business, with the stages read off how the business says things progress, the form each need collects, and how much each person will tolerate.\n\n## Intake\n\n{intake}\n{feedback}\nReply with the YAML only, no prose, no code fences."
|
||||
);
|
||||
let reply = crate::trainer::llm_with(&cmd, &prompt, &[("TEMPERATURE", "0.2")])?;
|
||||
std::fs::write(out.join(format!("llm/{round:02}-prompt.md")), &prompt)?;
|
||||
std::fs::write(out.join(format!("llm/{round:02}-reply.md")), &reply)?;
|
||||
let yaml = extract_yaml(&reply);
|
||||
match check(&yaml) {
|
||||
Ok(plan) if !plan.spectators().is_empty() && !spectators_sent && round < rounds => {
|
||||
spectators_sent = true;
|
||||
let findings = plan.spectators().join("\n- ");
|
||||
eprintln!("plan round {round}: the plan parses, and a role the opener speaks for has no move after sending; asking once more");
|
||||
feedback = format!("\n## Your last attempt parsed, with one thing to fix\n\n- {findings}\n\nGive each such need a move the person makes themselves (`moves_by: {{<stage>: [submitter, <group>]}}`, with a required `email` in its form) or `watched_by: [public]`, and return the whole plan again.\n\n```yaml\n{yaml}\n```\n");
|
||||
continue;
|
||||
}
|
||||
Ok(plan) => {
|
||||
let mut text = yaml.trim_end().to_string();
|
||||
text.push('\n');
|
||||
// The attached personas were written against someone
|
||||
// else's need ids. The plan is final by now, so a model
|
||||
// may map them to this plan's ids without its words
|
||||
// reaching any page: it returns ids and nothing else,
|
||||
// and an id not in the plan is ignored. What it leaves
|
||||
// unmapped is listed for the reader.
|
||||
if let Some(list) = personas.as_mut() {
|
||||
let mapped = map_personas(&cmd, &plan, list)?;
|
||||
if mapped > 0 {
|
||||
eprintln!("plan round {round}: {mapped} persona(s) mapped to this plan's needs");
|
||||
}
|
||||
text.push_str("\n# Held out from every model that writes words.\n");
|
||||
text.push_str(&serde_yaml::to_string(&serde_yaml::Mapping::from_iter([(
|
||||
serde_yaml::Value::from("personas"),
|
||||
list.clone(),
|
||||
)]))?);
|
||||
}
|
||||
let whole: Plan =
|
||||
serde_yaml::from_str(&text).context("the plan with its personas attached")?;
|
||||
let strangers: Vec<String> = whole
|
||||
.personas
|
||||
.iter()
|
||||
.filter(|p| {
|
||||
!whole
|
||||
.needs
|
||||
.iter()
|
||||
.any(|n| n.id == p["need"].as_str().unwrap_or_default())
|
||||
})
|
||||
.map(|p| {
|
||||
format!(
|
||||
"{} (need: {})",
|
||||
p["id"].as_str().unwrap_or_default(),
|
||||
p["need"].as_str().unwrap_or_default()
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
std::fs::write(out.join("plan.yaml"), &text)?;
|
||||
let groups: std::collections::BTreeSet<&str> = plan
|
||||
.needs
|
||||
.iter()
|
||||
.flat_map(|n| [Some(n.handled_by.as_str()), n.enters_as.as_deref()])
|
||||
.flatten()
|
||||
.collect();
|
||||
let buckets: std::collections::BTreeSet<&str> =
|
||||
plan.needs.iter().map(|n| n.lands_in.as_str()).collect();
|
||||
eprintln!("plan round {round}: {} need(s), {} group(s), {} bucket(s), {} persona(s) attached", plan.needs.len(), groups.len(), buckets.len(), whole.personas.len());
|
||||
for n in &plan.needs {
|
||||
eprintln!(
|
||||
" {}: {} -> {} ({}), {} of {} field(s) required",
|
||||
n.id,
|
||||
n.stages.join(" -> "),
|
||||
n.lands_in,
|
||||
n.handled_by,
|
||||
n.collects.iter().filter(|f| !f.optional).count(),
|
||||
n.max_fields
|
||||
);
|
||||
}
|
||||
if !strangers.is_empty() {
|
||||
eprintln!(" {} persona(s) name a need this plan does not have; set their `need:` to one of the ids above before building:\n {}", strangers.len(), strangers.join("\n "));
|
||||
}
|
||||
// A budget that equals the form is the tell of a model
|
||||
// that counted the form and called it the person's
|
||||
// tolerance. One such need is a form that uses its
|
||||
// room; most of them is the number being invented.
|
||||
let tight: Vec<&str> = plan
|
||||
.needs
|
||||
.iter()
|
||||
.filter(|n| n.collects.iter().filter(|f| !f.optional).count() == n.max_fields)
|
||||
.map(|n| n.id.as_str())
|
||||
.collect();
|
||||
if tight.len() * 2 > plan.needs.len() && tight.len() > 1 {
|
||||
eprintln!(" {} of {} need(s) have max_fields equal to their required fields ({}): check each against what the intake says about that person, not the form", tight.len(), plan.needs.len(), tight.join(", "));
|
||||
}
|
||||
println!(
|
||||
"plan written to {}; read it before building from it",
|
||||
out.join("plan.yaml").display()
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("plan round {round}: {e}");
|
||||
feedback = format!("\n## Your last attempt was refused\n\n{e}\n\nFix that and return the whole plan again.\n\n```yaml\n{yaml}\n```\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
bail!(
|
||||
"no plan passed in {rounds} round(s); the replies are in {}/llm/",
|
||||
out.display()
|
||||
)
|
||||
}
|
||||
|
||||
/// Personas whose `need:` is not one of the plan's ids get one mapped
|
||||
/// by a model that sees the plan's needs and the persona texts and
|
||||
/// returns ids only. Returns how many were mapped; an id the plan does
|
||||
/// not have, or a persona the model leaves out, stays as it was.
|
||||
fn map_personas(cmd: &str, plan: &Plan, personas: &mut serde_yaml::Value) -> anyhow::Result<usize> {
|
||||
let ids: Vec<&str> = plan.needs.iter().map(|n| n.id.as_str()).collect();
|
||||
let Some(list) = personas.as_sequence_mut() else { return Ok(0) };
|
||||
let strangers: Vec<(String, String)> = list
|
||||
.iter()
|
||||
.filter(|p| !ids.contains(&p["need"].as_str().unwrap_or_default()))
|
||||
.map(|p| (p["id"].as_str().unwrap_or_default().to_string(), p["text"].as_str().unwrap_or_default().to_string()))
|
||||
.filter(|(id, _)| !id.is_empty())
|
||||
.collect();
|
||||
if strangers.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let needs: Vec<String> = plan.needs.iter().map(|n| format!("- {}: {} - wants {}", n.id, n.who, n.wants)).collect();
|
||||
let people: Vec<String> = strangers.iter().map(|(id, text)| format!("- {id}: {text}")).collect();
|
||||
let prompt = format!(
|
||||
"A site has these needs, one per kind of visitor:\n{}\n\nThese people arrive at it:\n{}\n\nFor each person, which need is theirs? Reply with a JSON object mapping each person's id to one need id from the list, and nothing else. Leave a person out if no need fits.",
|
||||
needs.join("\n"),
|
||||
people.join("\n")
|
||||
);
|
||||
let reply = crate::trainer::llm_with(cmd, &prompt, &[("FORMAT", "json"), ("TEMPERATURE", "0.0")])?;
|
||||
let Some(mapping) = first_object(&reply) else { return Ok(0) };
|
||||
let mut mapped = 0;
|
||||
for persona in list.iter_mut() {
|
||||
let id = persona["id"].as_str().unwrap_or_default().to_string();
|
||||
if let Some(need) = mapping.get(&id).and_then(|v| v.as_str()) {
|
||||
if ids.contains(&need) && !ids.contains(&persona["need"].as_str().unwrap_or_default()) {
|
||||
persona["need"] = serde_yaml::Value::from(need);
|
||||
mapped += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(mapped)
|
||||
}
|
||||
|
||||
/// Parsed and validated exactly as `build` will.
|
||||
fn check(yaml: &str) -> anyhow::Result<Plan> {
|
||||
let plan: Plan =
|
||||
serde_yaml::from_str(yaml).map_err(|e| anyhow!("does not parse as a plan: {e}"))?;
|
||||
if !plan.personas.is_empty() {
|
||||
bail!("contains personas; write none");
|
||||
}
|
||||
if plan.needs.is_empty() {
|
||||
bail!("has no needs");
|
||||
}
|
||||
plan.validate()?;
|
||||
Ok(plan)
|
||||
}
|
||||
|
||||
/// The YAML in a reply: inside code fences if the model used them,
|
||||
/// otherwise from the first top-level key on. A stray JSON object
|
||||
/// (some models answer `{"business": ...}`) is turned into YAML.
|
||||
fn extract_yaml(reply: &str) -> String {
|
||||
let text = reply.trim();
|
||||
if let Some(start) = text.find("```") {
|
||||
let after = &text[start + 3..];
|
||||
let after = after.find('\n').map_or(after, |i| &after[i + 1..]);
|
||||
let body = after.find("```").map_or(after, |i| &after[..i]);
|
||||
return body.to_string();
|
||||
}
|
||||
if text.starts_with('{') {
|
||||
if let Some(v) = first_object(text) {
|
||||
return serde_yaml::to_string(&v).unwrap_or_default();
|
||||
}
|
||||
}
|
||||
let start = text
|
||||
.lines()
|
||||
.position(|l| {
|
||||
l.starts_with("business:") || l.starts_with("needs:") || l.starts_with("contact:")
|
||||
})
|
||||
.unwrap_or(0);
|
||||
text.lines().skip(start).collect::<Vec<_>>().join("\n")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const GOOD: &str = "business: B\ncontact: { name: I, email: i@b.example }\nneeds:\n - id: enquiry\n who: an owner\n wants: a look\n lands_in: enquiries\n handled_by: project_managers\n stages: [open, quoted, won, lost]\n done_when: [won, lost]\n success: [won]\n max_fields: 3\n collects:\n - { name: name }\n - { name: phone, type: tel }\n";
|
||||
|
||||
#[test]
|
||||
fn yaml_is_taken_from_fences_prose_or_json() {
|
||||
assert_eq!(
|
||||
extract_yaml(&format!("Here you go:\n```yaml\n{GOOD}```\nDone.")),
|
||||
GOOD
|
||||
);
|
||||
assert_eq!(
|
||||
extract_yaml(&format!("Sure, the plan:\n\n{GOOD}")),
|
||||
GOOD.trim_end()
|
||||
);
|
||||
let from_json = extract_yaml("{\"business\": \"B\", \"needs\": []}");
|
||||
assert!(from_json.contains("business: B"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_plan_is_held_to_the_same_checks_as_build() {
|
||||
assert!(check(GOOD).is_ok());
|
||||
assert!(
|
||||
check("business: B\ncontact: { name: I, email: i@b.example }\nneeds: []\n")
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("no needs")
|
||||
);
|
||||
assert!(check(&GOOD.replace("max_fields: 3", "max_fields: 1"))
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("tolerate 1"));
|
||||
assert!(check(&format!(
|
||||
"{GOOD}personas:\n - {{ id: p, need: enquiry, text: hi }}\n"
|
||||
))
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("personas"));
|
||||
assert!(check(&GOOD.replace(
|
||||
"lands_in: enquiries",
|
||||
"lands_in: henvendelser\n extra: 1"
|
||||
))
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("does not parse"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,635 @@
|
||||
//! Understanding, readability, friendliness: three scores to iterate
|
||||
//! wording against, each measured the way that keeps it from sticking
|
||||
//! at its maximum.
|
||||
//!
|
||||
//! - **Readability** is counted, not judged: LIX, sentence length plus
|
||||
//! the share of words over six letters, over the page's prose only -
|
||||
//! the question, the descriptions and the followup - never headings
|
||||
//! or field labels, which are not sentences and would drag the score
|
||||
//! down to "very easy" whatever the prose says. Below 30 reads very
|
||||
//! easily, 30-40 easily, 40-50 takes effort, above 50 is official
|
||||
//! prose.
|
||||
//! - **Understanding** is three yes-or-no questions put to each persona
|
||||
//! about the page in front of them: which option is theirs, what the
|
||||
//! form will ask, and what happens after sending. The last needs the
|
||||
//! followup page, which `rate` reads from the content repo (`--repo`)
|
||||
//! because the sim export stops at the form. `rate` also puts all
|
||||
//! three to a deliberately gutted twin of the page (headings replaced
|
||||
//! by "Option 1", descriptions, the form line and the followup
|
||||
//! removed). An instrument that still says yes to the twin is not
|
||||
//! measuring the page, and `rate` says so, per question. Every page
|
||||
//! on a need's path is rated: behind a door, "which" is decided on
|
||||
//! the door page and the form's page carries "asked" and "next".
|
||||
//! - **Friendliness** is a preference between two versions, never an
|
||||
//! absolute mark: `prefer` shows each persona both versions in both
|
||||
//! orders; a pick that flips with the order counts as no preference.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{anyhow, Context};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::trainer::card;
|
||||
use crate::trainer::simulate::ask;
|
||||
|
||||
fn flag(argv: &[String], name: &str) -> Option<String> {
|
||||
argv.iter().position(|a| a == name).and_then(|i| argv.get(i + 1)).cloned()
|
||||
}
|
||||
|
||||
fn load(path: &str) -> anyhow::Result<Value> {
|
||||
Ok(serde_json::from_str(&std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?)?)
|
||||
}
|
||||
|
||||
/// LIX = words per sentence + 100 * long words / words.
|
||||
pub fn lix(text: &str) -> f64 {
|
||||
let words: Vec<&str> = text
|
||||
.split(|c: char| !(c.is_alphanumeric() || c == '\'' || c == '-'))
|
||||
.filter(|w| w.chars().any(char::is_alphabetic))
|
||||
.collect();
|
||||
if words.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let sentences = text.split(['.', '!', '?', ':', '\n']).filter(|s| s.chars().any(char::is_alphabetic)).count().max(1);
|
||||
let long = words.iter().filter(|w| w.chars().count() > 6).count();
|
||||
words.len() as f64 / sentences as f64 + 100.0 * long as f64 / words.len() as f64
|
||||
}
|
||||
|
||||
pub fn lix_band(score: f64) -> &'static str {
|
||||
match score {
|
||||
s if s < 30.0 => "very easy",
|
||||
s if s < 40.0 => "easy",
|
||||
s if s < 50.0 => "takes effort",
|
||||
_ => "official prose",
|
||||
}
|
||||
}
|
||||
|
||||
/// The page as the visitor meets it: the question, every alternative,
|
||||
/// the form behind the one that serves them, and the page they land on
|
||||
/// after sending, when the repo is there to read it from.
|
||||
fn page_text(step: &Value, page: Option<&Value>, followup: Option<&Value>, gutted: bool) -> String {
|
||||
let mut text = match (gutted, page) {
|
||||
(false, Some(p)) => card::page_head(p),
|
||||
_ => format!("{}\n", step["question"].as_str().unwrap_or_default()),
|
||||
};
|
||||
// The cards that lead nowhere (the opener, the look around) are
|
||||
// read before the options; the gutted twin loses them too.
|
||||
if let Some(context) = step["context"].as_str().filter(|c| !c.trim().is_empty() && !gutted) {
|
||||
text.push_str(&format!("\n{context}\n"));
|
||||
}
|
||||
for (i, option) in step["options"].as_array().into_iter().flatten().enumerate() {
|
||||
if gutted {
|
||||
text.push_str(&format!("\n- Option {}\n", i + 1));
|
||||
continue;
|
||||
}
|
||||
// The whole card when the repo is there to read it from;
|
||||
// heading and description otherwise.
|
||||
let card = page
|
||||
.and_then(|p| p["alternatives"].as_array())
|
||||
.and_then(|alts| alts.iter().find(|a| a["name"] == option["name"]))
|
||||
.map(|a| card::alternative(a, true));
|
||||
match card {
|
||||
Some(c) => text.push_str(&format!("\n{c}")),
|
||||
None => text.push_str(&format!(
|
||||
"\n- {}\n {}\n",
|
||||
option["name"].as_str().unwrap_or_default(),
|
||||
option["description"].as_str().unwrap_or_default()
|
||||
)),
|
||||
}
|
||||
}
|
||||
if page.is_some() && !gutted {
|
||||
// The cards carry their own "Asks for" lines.
|
||||
return finish_page_text(text, followup);
|
||||
}
|
||||
// The twin has no form line at all: "Field 1, Field 2" still
|
||||
// tells a reader how many things they will be asked, and a model
|
||||
// answers "can you tell what you will be asked" with yes to that
|
||||
// (measured: 2 of 2 on every need).
|
||||
let fields: Vec<&str> = step["fields"].as_array().into_iter().flatten().filter_map(|f| f["label"].as_str()).collect();
|
||||
if !gutted && !fields.is_empty() {
|
||||
text.push_str(&format!(
|
||||
"\nThe form under \"{}\" asks for: {}.\n",
|
||||
step["alternative"].as_str().unwrap_or_default(),
|
||||
fields.join(", ")
|
||||
));
|
||||
}
|
||||
if gutted {
|
||||
return text;
|
||||
}
|
||||
finish_page_text(text, followup)
|
||||
}
|
||||
|
||||
fn finish_page_text(mut text: String, followup: Option<&Value>) -> String {
|
||||
if let Some(next) = followup {
|
||||
text.push_str(&format!("\nAfter you send it, the next page says:\n{}\n", next["name"].as_str().unwrap_or_default()));
|
||||
if let Some(d) = next["description"].as_str().filter(|d| !d.trim().is_empty()) {
|
||||
text.push_str(&format!("{}\n", d.trim()));
|
||||
}
|
||||
for alt in next["alternatives"].as_array().into_iter().flatten() {
|
||||
let d = alt["description"].as_str().unwrap_or_default().trim();
|
||||
text.push_str(&format!("- {}{}{}\n", alt["name"].as_str().unwrap_or_default(), if d.is_empty() { "" } else { ": " }, d));
|
||||
}
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
/// The running text a reader has to parse: the question, each
|
||||
/// description, and the followup. Headings and labels are left out.
|
||||
fn prose_text(step: &Value, followup: Option<&Value>) -> String {
|
||||
let mut prose = vec![step["question"].as_str().unwrap_or_default().to_string()];
|
||||
prose.extend(step["options"].as_array().into_iter().flatten().filter_map(|o| o["description"].as_str()).map(str::to_string));
|
||||
if let Some(next) = followup {
|
||||
prose.extend(next["description"].as_str().map(str::to_string));
|
||||
prose.extend(next["alternatives"].as_array().into_iter().flatten().filter_map(|a| a["description"].as_str()).map(str::to_string));
|
||||
}
|
||||
prose.into_iter().filter(|p| !p.trim().is_empty()).collect::<Vec<_>>().join("\n")
|
||||
}
|
||||
|
||||
/// The page a visitor lands on after the step's alternative: read from
|
||||
/// the content repo, since the sim export stops at the form. `None`
|
||||
/// when there is no repo, no `action`, or the page is not there.
|
||||
fn followup_page(repo: Option<&Path>, step: &Value) -> Option<Value> {
|
||||
let repo = repo?;
|
||||
let page = load_page(repo, step["page"].as_str()?)?;
|
||||
let alternative = step["alternative"].as_str()?;
|
||||
let action = page["alternatives"].as_array()?.iter().find(|a| a["name"] == alternative)?["action"].as_str()?;
|
||||
load_page(repo, action)
|
||||
}
|
||||
|
||||
/// Every desk's buttons per bucket, from the simulation model:
|
||||
/// (page, group, bucket, [(from, to, label)]). A desk with one button
|
||||
/// for a bucket has nothing to confuse and is left out.
|
||||
fn desk_buttons(model: &Value) -> Vec<(String, String, String, Vec<(String, String, String)>)> {
|
||||
let mut out = Vec::new();
|
||||
for (bucket, spec) in model["buckets"].as_object().into_iter().flatten() {
|
||||
for desk in spec["desks"].as_array().into_iter().flatten() {
|
||||
let moves: Vec<(String, String, String)> = desk["moves"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|m| (s(&m["from"]), s(&m["to"]), s(&m["label"])))
|
||||
.filter(|(_, _, l)| !l.is_empty())
|
||||
.collect();
|
||||
if moves.len() >= 2 {
|
||||
out.push((s(&desk["page"]), s(&desk["group"]), bucket.clone(), moves));
|
||||
}
|
||||
}
|
||||
}
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
|
||||
fn s(v: &Value) -> String {
|
||||
v.as_str().unwrap_or_default().to_string()
|
||||
}
|
||||
|
||||
/// A worker on the desk, told what just happened to a case, picks
|
||||
/// the button for it. The outcome is put in plain words from the
|
||||
/// state's own name, never from the label, and the buttons are listed
|
||||
/// in a seeded shuffle.
|
||||
fn button_pick(cmd: &str, group: &str, bucket: &str, from: &str, to: &str, labels: &[&str], seed: usize, words: &Value) -> anyhow::Result<String> {
|
||||
// The state in the plan's own words when the spec carries them,
|
||||
// else the id made readable.
|
||||
let plain = |id: &str| words[id].as_str().map(str::to_string).unwrap_or_else(|| id.replace(['_', '-'], " "));
|
||||
let mut order: Vec<&str> = labels.to_vec();
|
||||
// A fixed rotation per move: the right answer is not always first.
|
||||
let by = seed % order.len().max(1);
|
||||
order.rotate_left(by);
|
||||
let listed: Vec<String> = order.iter().enumerate().map(|(i, l)| format!("{}. {l}", i + 1)).collect();
|
||||
let prompt = format!(
|
||||
"You work on the {} desk of a small organisation. A case in \"{}\" was \"{}\" and has just become \"{}\". The desk shows these buttons for it:\n\n{}\n\nWhich button records what just happened? Reply as JSON: {{\"choice\": <the button's number>}}",
|
||||
plain(group),
|
||||
plain(bucket),
|
||||
plain(from),
|
||||
plain(to),
|
||||
listed.join("\n")
|
||||
);
|
||||
let reply = ask(cmd, &prompt, &seed.to_string())?;
|
||||
let n = reply["choice"].as_u64().unwrap_or(0) as usize;
|
||||
Ok(order.get(n.wrapping_sub(1)).map(|l| l.to_string()).unwrap_or_default())
|
||||
}
|
||||
|
||||
/// Every `mail:` on a state in the repo's aggregates.yaml, as
|
||||
/// (`bucket:state`, subject, body) with the `{placeholders}` taken
|
||||
/// out, so they read as the person will read them.
|
||||
pub(crate) fn mail_texts(repo: &Path) -> Option<Vec<(String, String, String)>> {
|
||||
let raw = std::fs::read_to_string(repo.join("aggregates.yaml")).ok()?;
|
||||
let doc: Value = serde_yaml::from_str(&raw).ok()?;
|
||||
let strip = |s: &str| {
|
||||
let mut out = String::new();
|
||||
let mut skip = false;
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'{' => skip = true,
|
||||
'}' => skip = false,
|
||||
_ if !skip => out.push(c),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
out.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
};
|
||||
Some(raw_mails(repo, &doc).into_iter().map(|(name, subject, body)| (name, strip(&subject), strip(&body))).collect())
|
||||
}
|
||||
|
||||
/// Every mail the site sends, as written: the state mails in
|
||||
/// aggregates.yaml, then the invite mail in site.yaml - onboarding is
|
||||
/// a mail too, and the first one a new member reads.
|
||||
fn raw_mails(repo: &Path, aggregates: &Value) -> Vec<(String, String, String)> {
|
||||
let mut out = Vec::new();
|
||||
for aggregate in aggregates["aggregates"].as_array().into_iter().flatten() {
|
||||
let bucket = aggregate["bucket"].as_str().unwrap_or_default();
|
||||
for (state, spec) in aggregate["states"].as_object().into_iter().flatten() {
|
||||
if let Some(mail) = spec.get("mail") {
|
||||
out.push((
|
||||
format!("{bucket}:{state}"),
|
||||
mail["subject"].as_str().unwrap_or_default().to_string(),
|
||||
mail["body"].as_str().unwrap_or_default().to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
let site: Value = std::fs::read_to_string(repo.join("site.yaml")).ok().and_then(|t| serde_yaml::from_str(&t).ok()).unwrap_or(Value::Null);
|
||||
let invite = &site["mail"]["invite"];
|
||||
if invite.is_object() {
|
||||
out.push(("invite".to_string(), invite["subject"].as_str().unwrap_or_default().to_string(), invite["body"].as_str().unwrap_or_default().to_string()));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A mail as it arrives: the placeholders filled with plausible values,
|
||||
/// so a reader meets a link where the link will be, not a blank.
|
||||
fn as_received(template: &str, site_name: &str) -> String {
|
||||
let mut out = String::new();
|
||||
let mut rest = template;
|
||||
while let Some(i) = rest.find('{') {
|
||||
out.push_str(&rest[..i]);
|
||||
let after = &rest[i + 1..];
|
||||
match after.find('}') {
|
||||
Some(j) if after[..j].chars().all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.') && j > 0 => {
|
||||
out.push_str(&match &after[..j] {
|
||||
"site" => "https://example.site".to_string(),
|
||||
"chain" => "8f3a1c".to_string(),
|
||||
"link" => "https://id.example.site/reset/8f3a1c".to_string(),
|
||||
"site_name" => site_name.to_string(),
|
||||
"expires" => "2026-10-05 12:00 UTC".to_string(),
|
||||
key => format!("[{}]", key.replace('_', " ")),
|
||||
});
|
||||
rest = &after[j + 1..];
|
||||
}
|
||||
_ => {
|
||||
out.push('{');
|
||||
rest = after;
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push_str(rest);
|
||||
out
|
||||
}
|
||||
|
||||
const MAIL_QUESTIONS: [(&str, &str); 3] = [
|
||||
("happened", "Do you know, from this mail alone, what has happened and to what?"),
|
||||
("now", "Do you know what, if anything, you can or must do now, and how?"),
|
||||
("who", "Do you know who sent it, and how you would answer them?"),
|
||||
];
|
||||
|
||||
/// Three yes-or-no questions on one mail, from the person it is for.
|
||||
fn mail_understanding(cmd: &str, reader: &str, subject: &str, body: &str, seed: usize) -> anyhow::Result<[bool; 3]> {
|
||||
let prompt = format!(
|
||||
"You are {reader}. This mail has just arrived:\n---\nSubject: {subject}\n\n{body}\n---\n\nAnswer three questions as yourself. Say yes only when the mail itself tells you; if you would be guessing, say no.\n{}\nReply as JSON: {{\"happened\": true or false, \"now\": true or false, \"who\": true or false}}",
|
||||
MAIL_QUESTIONS.iter().map(|(k, q)| format!("- {k}: {q}")).collect::<Vec<_>>().join("\n")
|
||||
);
|
||||
let reply = ask(cmd, &prompt, &seed.to_string())?;
|
||||
Ok([reply["happened"] == true, reply["now"] == true, reply["who"] == true])
|
||||
}
|
||||
|
||||
/// `/` is `questions/index.yaml`; `/a/b` is `questions/a/b.yaml` or
|
||||
/// `questions/a/b/index.yaml`.
|
||||
pub(crate) fn load_page(repo: &Path, route: &str) -> Option<Value> {
|
||||
let rel = route.trim_matches('/');
|
||||
let candidates = if rel.is_empty() {
|
||||
vec![repo.join("questions/index.yaml")]
|
||||
} else {
|
||||
vec![repo.join(format!("questions/{rel}.yaml")), repo.join(format!("questions/{rel}/index.yaml"))]
|
||||
};
|
||||
let text = candidates.iter().find_map(|p| std::fs::read_to_string(p).ok())?;
|
||||
serde_yaml::from_str(&text).ok()
|
||||
}
|
||||
|
||||
/// What one page may ask of one person before it should be split into
|
||||
/// a further question, or the fields moved to a later stage of the
|
||||
/// case where they are actually needed. Complexity belongs in the
|
||||
/// number of steps and states, not in the weight of any one page.
|
||||
const MAX_OPTIONS: usize = 5;
|
||||
const MAX_REQUIRED: usize = 5;
|
||||
|
||||
/// (alternatives offered, required fields in the form that serves them)
|
||||
fn page_load(step: &Value) -> (usize, usize) {
|
||||
(
|
||||
step["options"].as_array().map_or(0, Vec::len),
|
||||
step["fields"].as_array().into_iter().flatten().filter(|f| f["optional"] != true).count(),
|
||||
)
|
||||
}
|
||||
|
||||
const QUESTIONS: [(&str, &str); 4] = [
|
||||
("which", "Is it clear from the page which option is meant for someone in your situation?"),
|
||||
("where", "Do you know where you are on this site, who this option is for, and what pressing its button will do?"),
|
||||
("asked", "Can you tell from the page what you will be asked to provide?"),
|
||||
("next", "Does the page tell you what happens after you send it, or who will read it?"),
|
||||
];
|
||||
const N: usize = QUESTIONS.len();
|
||||
|
||||
fn understanding(cmd: &str, who: &str, page: &str, seed: usize) -> anyhow::Result<[bool; N]> {
|
||||
let prompt = format!(
|
||||
"You are this person: {who}\n\nThis is the whole of a web page you have just opened:\n---\n{page}---\n\nAnswer four questions about it as yourself. Say yes only when the page text itself gives you that; if you would be guessing, say no.\n{}\nReply as JSON: {{\"which\": true or false, \"where\": true or false, \"asked\": true or false, \"next\": true or false}}",
|
||||
QUESTIONS.iter().map(|(k, q)| format!("- {k}: {q}")).collect::<Vec<_>>().join("\n")
|
||||
);
|
||||
let reply = ask(cmd, &prompt, &seed.to_string())?;
|
||||
Ok([reply["which"] == true, reply["where"] == true, reply["asked"] == true, reply["next"] == true])
|
||||
}
|
||||
|
||||
pub fn rate(argv: &[String]) -> anyhow::Result<()> {
|
||||
let usage = "usage: portal-trainer rate --model <model.json> --out <dir> [--repo <content dir>] [--llm 'cmd']";
|
||||
let model = load(&flag(argv, "--model").ok_or_else(|| anyhow!(usage))?)?;
|
||||
let out = std::path::PathBuf::from(flag(argv, "--out").ok_or_else(|| anyhow!(usage))?);
|
||||
let repo = flag(argv, "--repo").map(std::path::PathBuf::from);
|
||||
let cmd = flag(argv, "--llm").unwrap_or_else(|| "claude -p".to_string());
|
||||
std::fs::create_dir_all(&out)?;
|
||||
if repo.is_none() {
|
||||
eprintln!("rate: no --repo, so no followup page is shown and \"next\" is not measured");
|
||||
}
|
||||
|
||||
let mut report = BTreeMap::new();
|
||||
// The desks: the inside of the business reads button labels and
|
||||
// nothing else, so each desk is scored on whether a worker told
|
||||
// what has just happened to a case picks the right button among
|
||||
// that desk's buttons for the bucket.
|
||||
let desks = desk_buttons(&model);
|
||||
if !desks.is_empty() {
|
||||
println!("{:<20} {:<24} {:>8} {}", "desk", "bucket", "buttons", "mistaken for");
|
||||
for (page, group, bucket, moves) in &desks {
|
||||
// A desk row shows only the buttons that start from the
|
||||
// row's state, so those are the ones the worker chooses
|
||||
// among; a state with one button has nothing to confuse.
|
||||
let mut right = 0;
|
||||
let mut asked = 0;
|
||||
let mut mistaken = Vec::new();
|
||||
for (i, (from, to, label)) in moves.iter().enumerate() {
|
||||
let labels: Vec<&str> = moves.iter().filter(|(f, _, _)| f == from).map(|(_, _, l)| l.as_str()).collect();
|
||||
if labels.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
asked += 1;
|
||||
let picked = button_pick(&cmd, group, bucket, from, to, &labels, i, &model["buckets"][bucket]["words"])?;
|
||||
if picked == *label {
|
||||
right += 1;
|
||||
} else {
|
||||
mistaken.push(format!("{from}>{to} as {picked:?}"));
|
||||
}
|
||||
}
|
||||
if asked == 0 {
|
||||
continue;
|
||||
}
|
||||
println!("{:<20} {:<24} {:>3} of {:<2} {}", page, bucket, right, asked, mistaken.join("; "));
|
||||
report.insert(format!("desk {page} {bucket}"), json!({ "right": right, "of": asked, "mistaken": mistaken }));
|
||||
}
|
||||
println!();
|
||||
}
|
||||
// The mails first, when the repo carries any: a person reads a
|
||||
// mail alone, with no card around it, so its prose is rated on
|
||||
// its own - LIX and length.
|
||||
// Then the three questions a person asks of any mail - what
|
||||
// happened, what now, who is this - each also put to a gutted twin
|
||||
// that says only that there is news. A yes the twin also gets is
|
||||
// the reader agreeing with anything.
|
||||
if let Some(mails) = repo.as_deref().and_then(mail_texts) {
|
||||
if !mails.is_empty() {
|
||||
let dir = repo.as_deref().unwrap_or(Path::new("."));
|
||||
let site: Value = std::fs::read_to_string(dir.join("site.yaml")).ok().and_then(|t| serde_yaml::from_str(&t).ok()).unwrap_or(Value::Null);
|
||||
let site_name = site["title"].as_str().unwrap_or("the site").to_string();
|
||||
let aggregates: Value = std::fs::read_to_string(dir.join("aggregates.yaml")).ok().and_then(|t| serde_yaml::from_str(&t).ok()).unwrap_or(Value::Null);
|
||||
let raw = raw_mails(dir, &aggregates);
|
||||
println!("{:<34} {:>5} {:<14} {:>5} {:>8} {:>5} {:>5} {:<16}", "mail", "LIX", "", "words", "happened", "now", "who", "gutted twin says");
|
||||
for (i, ((name, subject, body), (_, raw_subject, raw_body))) in mails.iter().zip(&raw).enumerate() {
|
||||
let text = format!("{subject}. {body}");
|
||||
let score = lix(&text);
|
||||
let reader = if name == "invite" {
|
||||
format!("someone a member of {site_name} has just brought in to help run it; you did not ask for this mail")
|
||||
} else {
|
||||
format!("someone who sent a form to {site_name} some time ago")
|
||||
};
|
||||
let (subject_in, body_in) = (as_received(raw_subject, &site_name), as_received(raw_body, &site_name));
|
||||
let got = mail_understanding(&cmd, &reader, &subject_in, &body_in, i)?;
|
||||
let twin = mail_understanding(&cmd, &reader, "News", "There is news.", i)?;
|
||||
let yn = |b: bool| if b { "yes" } else { "no" };
|
||||
let failed: Vec<&str> = MAIL_QUESTIONS.iter().zip(twin).filter(|(_, t)| *t).map(|((k, _), _)| *k).collect();
|
||||
println!(
|
||||
"{:<34} {:>5.0} {:<14} {:>5} {:>8} {:>5} {:>5} {:<16}",
|
||||
name, score, lix_band(score), text.split_whitespace().count(), yn(got[0]), yn(got[1]), yn(got[2]),
|
||||
if failed.is_empty() { "ok".to_string() } else { format!("FAILED: {}", failed.join(", ")) }
|
||||
);
|
||||
report.insert(format!("mail {name}"), json!({
|
||||
"lix": score, "words": text.split_whitespace().count(),
|
||||
"understanding": { "happened": got[0], "now": got[1], "who": got[2] },
|
||||
"control_ok": failed.is_empty(),
|
||||
}));
|
||||
}
|
||||
println!();
|
||||
}
|
||||
}
|
||||
println!("{:<28} {:>6} {:<14} {:>8} {:>8} {:>8} {:>8} {:<22} {:<22}", "need", "LIX", "", "which", "where", "asked", "next", "gutted twin says", "page load");
|
||||
for need in model["needs"].as_array().into_iter().flatten() {
|
||||
let id = need["id"].as_str().unwrap_or_default();
|
||||
let steps = need["steps"].as_array().cloned().unwrap_or_default();
|
||||
let personas: Vec<&Value> = model["personas"].as_array().into_iter().flatten().filter(|p| p["need"] == id).collect();
|
||||
if personas.is_empty() {
|
||||
continue;
|
||||
}
|
||||
// Every page on the path is rated. The followup follows only
|
||||
// the last page's alternative: on a door page, what the
|
||||
// alternative leads to is the next page, not what happens
|
||||
// after sending.
|
||||
let mut pages = Vec::new();
|
||||
for (i, step) in steps.iter().enumerate() {
|
||||
let followup = if i + 1 == steps.len() { followup_page(repo.as_deref(), step) } else { None };
|
||||
let page = repo.as_deref().and_then(|r| load_page(r, step["page"].as_str().unwrap_or("/")));
|
||||
let row = if steps.len() > 1 { format!("{id} @{}", step["page"].as_str().unwrap_or_default()) } else { id.to_string() };
|
||||
let readability = lix(&prose_text(step, followup.as_ref()));
|
||||
let (mut yes, mut control) = ([0usize; N], [0usize; N]);
|
||||
for (i, persona) in personas.iter().enumerate() {
|
||||
let who = persona["text"].as_str().unwrap_or_default();
|
||||
for (tally, gutted) in [(&mut yes, false), (&mut control, true)] {
|
||||
let answers = understanding(&cmd, who, &page_text(step, page.as_ref(), followup.as_ref(), gutted), i)?;
|
||||
for (slot, answer) in tally.iter_mut().zip(answers) {
|
||||
*slot += answer as usize;
|
||||
}
|
||||
}
|
||||
}
|
||||
let n = personas.len();
|
||||
let measured = |k: usize| k != 3 || followup.is_some();
|
||||
let share = |k: usize| if measured(k) { format!("{}/{n}", yes[k]) } else { "-".to_string() };
|
||||
// The twin has no headings, no descriptions, no cards and no
|
||||
// followup: every question must collapse on it, or that
|
||||
// question is agreeing with anything.
|
||||
let control_ok: Vec<bool> = (0..N).map(|k| !measured(k) || control[k] * 2 < n.max(1)).collect();
|
||||
let failed: Vec<&str> = QUESTIONS.iter().zip(&control_ok).filter(|(_, ok)| !**ok).map(|((k, _), _)| *k).collect();
|
||||
let (options, required) = page_load(step);
|
||||
println!(
|
||||
"{:<28} {:>6.0} {:<14} {:>8} {:>8} {:>8} {:>8} {:<22} {:<22}",
|
||||
row,
|
||||
readability,
|
||||
lix_band(readability),
|
||||
share(0),
|
||||
share(1),
|
||||
share(2),
|
||||
share(3),
|
||||
format!(
|
||||
"{}/{}/{}/{} {}",
|
||||
control[0],
|
||||
control[1],
|
||||
control[2],
|
||||
if measured(3) { control[3].to_string() } else { "-".to_string() },
|
||||
if failed.is_empty() { "ok".to_string() } else { format!("FAILED: {}", failed.join(", ")) }
|
||||
),
|
||||
format!(
|
||||
"{options} options, {required} required{}",
|
||||
if options > MAX_OPTIONS || required > MAX_REQUIRED { " - SPLIT" } else { "" }
|
||||
),
|
||||
);
|
||||
pages.push(json!({
|
||||
"page": step["page"], "lix": readability, "personas": n,
|
||||
"options": options, "required_fields": required,
|
||||
"overloaded": options > MAX_OPTIONS || required > MAX_REQUIRED,
|
||||
"followup_shown": followup.is_some(),
|
||||
"understanding": {"which": yes[0], "where": yes[1], "asked": yes[2], "next": if measured(3) { json!(yes[3]) } else { Value::Null }},
|
||||
"control": {"which": control[0], "where": control[1], "asked": control[2], "next": if measured(3) { json!(control[3]) } else { Value::Null }},
|
||||
"control_ok": failed.is_empty(),
|
||||
}));
|
||||
}
|
||||
report.insert(id.to_string(), json!({ "steps": pages.len(), "pages": pages }));
|
||||
}
|
||||
std::fs::write(out.join("rate.json"), serde_json::to_string_pretty(&report)?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn prefer(argv: &[String]) -> anyhow::Result<()> {
|
||||
let usage = "usage: portal-trainer prefer --a <model-a.json> --b <model-b.json> [--llm 'cmd']";
|
||||
let a = load(&flag(argv, "--a").ok_or_else(|| anyhow!(usage))?)?;
|
||||
let b = load(&flag(argv, "--b").ok_or_else(|| anyhow!(usage))?)?;
|
||||
let cmd = flag(argv, "--llm").unwrap_or_else(|| "claude -p".to_string());
|
||||
|
||||
println!("{:<28} {:>5} {:>5} {:>14} {:>5} {:>5} {:>14} {:>9}", "need", "A", "B", "no preference", "A", "B", "no preference", "LIX A->B");
|
||||
println!("{:<28} {:^26} {:^26}", "", "friendlier", "clearer");
|
||||
for need_a in a["needs"].as_array().into_iter().flatten() {
|
||||
let id = need_a["id"].as_str().unwrap_or_default();
|
||||
let Some(need_b) = b["needs"].as_array().into_iter().flatten().find(|n| n["id"] == id) else { continue };
|
||||
let (Some(step_a), Some(step_b)) = (need_a["steps"].as_array().and_then(|s| s.first()), need_b["steps"].as_array().and_then(|s| s.first())) else { continue };
|
||||
let (text_a, text_b) = (page_text(step_a, None, None, false), page_text(step_b, None, None, false));
|
||||
if text_a == text_b {
|
||||
continue;
|
||||
}
|
||||
// [friendlier, clearer] x [A, B, none]
|
||||
let mut wins = [[0usize; 3]; 2];
|
||||
for (i, persona) in a["personas"].as_array().into_iter().flatten().filter(|p| p["need"] == id).enumerate() {
|
||||
let who = persona["text"].as_str().unwrap_or_default();
|
||||
let mut picks: [[&str; 2]; 2] = [["", ""], ["", ""]];
|
||||
for (order, (first, second)) in [(&text_a, &text_b), (&text_b, &text_a)].into_iter().enumerate() {
|
||||
let prompt = format!(
|
||||
"You are this person: {who}\n\nTwo versions of the same web page. They lead to the same thing; only the wording differs.\n\n=== VERSION 1 ===\n{first}\n=== VERSION 2 ===\n{second}\n\nAs yourself: which version makes you feel more welcome and at ease, and which one makes it clearer what to do?\nReply as JSON: {{\"friendlier\": 1 or 2, \"clearer\": 1 or 2}}"
|
||||
);
|
||||
let reply = ask(&cmd, &prompt, &(i * 2 + order).to_string())?;
|
||||
for (k, key) in ["friendlier", "clearer"].iter().enumerate() {
|
||||
let first_won = reply[key] == 1;
|
||||
// In the second order, version 1 is B.
|
||||
picks[k][order] = if first_won == (order == 0) { "A" } else { "B" };
|
||||
}
|
||||
}
|
||||
for k in 0..2 {
|
||||
let slot = match picks[k] {
|
||||
["A", "A"] => 0,
|
||||
["B", "B"] => 1,
|
||||
_ => 2,
|
||||
};
|
||||
wins[k][slot] += 1;
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"{:<28} {:>5} {:>5} {:>14} {:>5} {:>5} {:>14} {:>9}",
|
||||
id, wins[0][0], wins[0][1], wins[0][2], wins[1][0], wins[1][1], wins[1][2],
|
||||
format!("{:.0}->{:.0}", lix(&text_a), lix(&text_b))
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn lix_separates_plain_words_from_official_prose() {
|
||||
let plain = "Get someone to look at a building. We come and see it. Then we tell you what it takes.";
|
||||
let official = "Submission of documentation regarding rehabilitation requirements necessitates comprehensive structural evaluation procedures.";
|
||||
assert!(lix(plain) < 30.0, "{}", lix(plain));
|
||||
assert!(lix(official) > 60.0, "{}", lix(official));
|
||||
assert_eq!(lix_band(lix(plain)), "very easy");
|
||||
assert_eq!(lix(""), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gutted_twin_keeps_the_question_and_loses_the_wording() {
|
||||
let step = json!({
|
||||
"question": "What brings you here?",
|
||||
"options": [{"name": "Get a price", "description": "For owners."}, {"name": "Work with us", "description": "For crews."}],
|
||||
"alternative": "Get a price",
|
||||
"fields": [{"label": "Email"}, {"label": "The build"}]
|
||||
});
|
||||
let next = json!({"name": "What happens now?", "description": "A project manager reads it.", "alternatives": [{"name": "We call you within two days", "description": ""}]});
|
||||
let page = page_text(&step, None, Some(&next), false);
|
||||
assert!(page.contains("Get a price") && page.contains("asks for: Email, The build"));
|
||||
assert!(page.contains("After you send it") && page.contains("A project manager reads it."));
|
||||
// With the repo's page in hand, each option is its whole card.
|
||||
let repo_page = json!({"name": "What brings you here?", "description": "", "alternatives": [
|
||||
{"name": "Get a price", "description": "For owners.", "consequence": ["Send it"], "encouragements": ["We answer within two days."], "features": [{"name": "Your brief", "requirements": [{"name": "email", "label": "Email"}]}]},
|
||||
{"name": "Work with us", "description": "For crews."}]});
|
||||
let rich = page_text(&step, Some(&repo_page), Some(&next), false);
|
||||
assert!(rich.contains("Button: Send it") && rich.contains("We answer within two days.") && rich.contains("Asks for: Email"));
|
||||
assert!(!rich.contains("asks for: Email, The build"));
|
||||
let twin = page_text(&step, Some(&repo_page), Some(&next), true);
|
||||
assert!(!twin.contains("Send it") && !twin.contains("Your brief"));
|
||||
let twin = page_text(&step, None, Some(&next), true);
|
||||
assert!(twin.contains("What brings you here?") && twin.contains("Option 2"));
|
||||
assert!(!twin.contains("asks for"));
|
||||
assert!(!twin.contains("Get a price") && !twin.contains("Email") && !twin.contains("project manager"));
|
||||
// Readability is measured on the sentences, not the headings.
|
||||
let prose = prose_text(&step, Some(&next));
|
||||
assert_eq!(prose, "What brings you here?\nFor owners.\nFor crews.\nA project manager reads it.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_mail_is_read_as_it_arrives_and_the_invite_is_one_of_them() {
|
||||
let got = as_received("Hei {name}, se {site}/decide/x?chain={chain}. {link} {ikke en nøkkel}", "Tomter Vel");
|
||||
assert_eq!(got, "Hei [name], se https://example.site/decide/x?chain=8f3a1c. https://id.example.site/reset/8f3a1c {ikke en nøkkel}");
|
||||
let dir = std::env::temp_dir().join(format!("portal-trainer-mails-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(dir.join("aggregates.yaml"), "aggregates:\n - bucket: b\n initial: open\n states:\n open: { event: received, mail: { to: submitter, subject: Fått, body: \"Vi har det, {name}.\" } }\n").unwrap();
|
||||
std::fs::write(dir.join("site.yaml"), "title: T\nmail: { from: x@y.no, invite: { subject: Konto, body: \"Hei {name}\\n{link}\" } }\n").unwrap();
|
||||
let mails = mail_texts(&dir).unwrap();
|
||||
assert_eq!(mails.iter().map(|m| m.0.as_str()).collect::<Vec<_>>(), ["b:open", "invite"]);
|
||||
assert_eq!(mails[0].2, "Vi har det, .");
|
||||
std::fs::remove_dir_all(dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_followup_page_is_read_from_the_repo_by_following_the_action() {
|
||||
let dir = std::env::temp_dir().join(format!("portal-trainer-rate-{}", std::process::id()));
|
||||
std::fs::create_dir_all(dir.join("questions/received")).unwrap();
|
||||
std::fs::write(dir.join("questions/index.yaml"), "name: What?\nalternatives:\n - name: Get a price\n action: /received/price\n").unwrap();
|
||||
std::fs::write(dir.join("questions/received/price.yaml"), "name: What happens now?\ndescription: We read it.\nalternatives:\n - name: Back\n").unwrap();
|
||||
let step = json!({"page": "/", "alternative": "Get a price"});
|
||||
let next = followup_page(Some(&dir), &step).unwrap();
|
||||
assert_eq!(next["name"], "What happens now?");
|
||||
assert!(followup_page(None, &step).is_none());
|
||||
assert!(followup_page(Some(&dir), &json!({"page": "/", "alternative": "Nope"})).is_none());
|
||||
std::fs::remove_dir_all(dir).ok();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
# Portal content reference (for the generator)
|
||||
|
||||
Portal serves a site that is one question per page. Each page offers a
|
||||
few alternatives. An alternative may carry a form; submitting it
|
||||
records the answer into a *bucket* whose records move through a state
|
||||
machine, worked from a *desk*: a page gated to a group, listing the
|
||||
bucket with one button per allowed move. There is no other mechanism.
|
||||
Everything a business needs is expressed as: who answers which
|
||||
question, where the answer lands, who moves it, through which states.
|
||||
|
||||
A content repo is these files and nothing else:
|
||||
|
||||
```
|
||||
needs.yaml what the business needs (the spec you are held to)
|
||||
aggregates.yaml one state machine per bucket
|
||||
site.yaml title and language
|
||||
questions/index.yaml the front door, "/"
|
||||
questions/<name>.yaml "/<name>"
|
||||
questions/<dir>/index.yaml "/<dir>"
|
||||
questions/<dir>/_section.yaml applies `qualifies` to the whole directory
|
||||
questions/<dir>/<name>.yaml "/<dir>/<name>" - a followup page by default
|
||||
```
|
||||
|
||||
## needs.yaml
|
||||
|
||||
```yaml
|
||||
needs:
|
||||
- id: kebab-case
|
||||
who: the kind of person, in plain words
|
||||
wants: what they want from the business
|
||||
enters: / # page they arrive on; default /
|
||||
enters_as: <group> # only when they are logged-in members of a group; omit for the public
|
||||
via: <alternative name> # only when several alternatives on the entry page record into the same bucket
|
||||
lands_in: <bucket>
|
||||
handled_by: <group> # the role inside the business that must see and move the record
|
||||
done_when: [<state>, <state>] # every state that counts as finished, good or bad
|
||||
max_steps: 1 # submissions before the record exists (1 or 2)
|
||||
max_fields: 4 # required fields the visitor will tolerate across the path
|
||||
success: [<state>] # the done_when states that are a good outcome for the business
|
||||
stages: [open, <stage>, <stage>, <end>, <end>]
|
||||
# EVERY stage the business said a case goes through, first to last,
|
||||
# read off their own sentences one verb at a time ("they come in, we
|
||||
# go and look, we send a quote, we win or lose it" is five stages).
|
||||
# The state machine must have a state for each, or the verifier fails.
|
||||
personas:
|
||||
- id: kebab-case
|
||||
need: <need id>
|
||||
text: >
|
||||
Two or three sentences in the visitor's own voice and vocabulary.
|
||||
```
|
||||
|
||||
Rules:
|
||||
- One need per (kind of person, thing they want). Internal work counts:
|
||||
a role inside the business recording or posting something is a need
|
||||
with `enters_as`.
|
||||
- States come from the business's own description of how a thing
|
||||
progresses. A pipeline is not an inbox: "answered" is rarely the real
|
||||
outcome.
|
||||
- `max_fields` is small on purpose. Reporting something from a phone
|
||||
gets 3. A first enquiry gets 4 to 6.
|
||||
- Two or three personas per public need. At least one must avoid the
|
||||
trade's own vocabulary. A persona must never quote a page heading.
|
||||
|
||||
## aggregates.yaml
|
||||
|
||||
```yaml
|
||||
aggregates:
|
||||
- bucket: leads
|
||||
initial: open
|
||||
states: # state: { event: past-tense verb }
|
||||
open: { event: received }
|
||||
quoted: { event: quoted }
|
||||
won: { event: won }
|
||||
lost: { event: lost }
|
||||
transitions: # only moves the business described
|
||||
open: [quoted, lost]
|
||||
quoted: [won, lost]
|
||||
```
|
||||
|
||||
The initial state should be `open` (a desk transition with no `from`
|
||||
means from `open`).
|
||||
|
||||
## A page
|
||||
|
||||
```yaml
|
||||
name: What brings you here? # always a question the visitor could ask themselves
|
||||
description: >
|
||||
One or two sentences.
|
||||
responsible: { name: <person or team>, contact: <email> } # required on every entry page
|
||||
alternatives:
|
||||
- name: <heading> # what the visitor picks. Must work WITHOUT the description.
|
||||
description: >
|
||||
One or two sentences.
|
||||
action: /received # absolute path of an existing page
|
||||
consequence: [Send] # the button label
|
||||
record_as: leads # omit for a pure navigation alternative
|
||||
features:
|
||||
- name: About the building
|
||||
description: Optional line of guidance.
|
||||
requirements:
|
||||
- { name: email, label: Email, type: email }
|
||||
- { name: problem, label: What is wrong, type: textarea }
|
||||
- { name: urgency, label: How soon, type: select, options: [This week, This month, No rush] }
|
||||
- { name: address, label: Address, optional: true }
|
||||
```
|
||||
|
||||
- Field types: `text` (default), `email`, `tel`, `textarea`, `date`,
|
||||
`select` (must list `options`). Use no other types.
|
||||
- Every `action` must name a page that exists. After a recording
|
||||
alternative, send the visitor to a followup page that says who reads
|
||||
it, when they answer, and what happens next. A followup page is a
|
||||
top-level file with `followup: true`, or any non-index file in a
|
||||
subdirectory. Give it one alternative with `action: /` to go back.
|
||||
- A page for members of a group sets `qualifies: <group>` (or sits in a
|
||||
directory whose `_section.yaml` does). Only `enters_as: <group>`
|
||||
needs can use it.
|
||||
- Every bucket that is recorded into must be read by a desk.
|
||||
|
||||
## A desk
|
||||
|
||||
```yaml
|
||||
# questions/desk/_section.yaml
|
||||
qualifies: project_managers
|
||||
```
|
||||
```yaml
|
||||
# questions/desk/index.yaml
|
||||
name: What came in?
|
||||
responsible: { name: ..., contact: ... }
|
||||
alternatives:
|
||||
- name: Leads
|
||||
description: One line on what to do with them.
|
||||
action: /desk
|
||||
consequence: [Confirm]
|
||||
features:
|
||||
- name: ""
|
||||
description: ""
|
||||
resource:
|
||||
source: { kind: kv, bucket: leads }
|
||||
requires_group: project_managers
|
||||
transitions:
|
||||
- { to: quoted, label: Quote sent }
|
||||
- { to: lost, label: Not for us }
|
||||
- { from: quoted, to: won, label: Won }
|
||||
- { from: quoted, to: lost, label: Lost }
|
||||
```
|
||||
|
||||
- One desk directory per handling group; one alternative per bucket
|
||||
that group handles.
|
||||
- Offer every move the state machine allows and no other. A state with
|
||||
no move out of it, on the way to a finished state, strands records.
|
||||
- A group may read a bucket it does not handle: the same `resource`
|
||||
block with its own `requires_group` and no `transitions`.
|
||||
|
||||
## site.yaml
|
||||
|
||||
```yaml
|
||||
title: <business name>
|
||||
lang: en
|
||||
```
|
||||
|
||||
## Language
|
||||
|
||||
Everything a person reads is written in the intake's `language`: page
|
||||
names, headings, descriptions, field labels, button labels, the
|
||||
`who`/`wants` of each need, and the personas. Write it the way people
|
||||
in that trade and country actually speak, not as a translation.
|
||||
Everything a machine reads stays plain ASCII snake_case or kebab-case:
|
||||
need ids, persona ids, bucket names, state names, field `name`s, file
|
||||
names and paths. Letters such as æ, ø and å belong in the copy and
|
||||
never in an identifier. Set `lang` in site.yaml to the intake's code.
|
||||
|
||||
## Copy rules
|
||||
|
||||
- Headings say what the visitor gets, in the visitor's words.
|
||||
- Lead with what a thing enables. Never define something by what it
|
||||
is not. No negations in headings.
|
||||
- Two alternatives must never be separable only by a distinction the
|
||||
business makes and the visitor does not.
|
||||
- Every required field is one the business will use to answer.
|
||||
|
||||
## The verifier
|
||||
|
||||
`question_lint` runs after every attempt. Non-zero exit means you are
|
||||
not done. Its messages are exact:
|
||||
|
||||
- `no path from ... records into ...`: no chain of alternatives from
|
||||
the entry page reaches an alternative with that `record_as`.
|
||||
- `the path asks for N required field(s), over the budget`: mark fields
|
||||
`optional: true` or drop them.
|
||||
- `no desk gated on "g" reads "b"`: add the desk.
|
||||
- `"g" cannot move a record to "s"` / `records strand in "s"`: the desk
|
||||
is missing a transition the state machine allows, or the state
|
||||
machine is missing the edge.
|
||||
- `action ... does not match any declared question id`: the page does
|
||||
not exist.
|
||||
- `entry page ... names no one responsible`: add `responsible`.
|
||||
@@ -0,0 +1,127 @@
|
||||
//! What a live site's storylines did, read back as counts.
|
||||
//!
|
||||
//! `portal-trainer report --site https://vel.example [--needs needs.yaml]`
|
||||
//! fetches the site's `/report` (portal docs, Storylines): per bucket,
|
||||
//! per storyline, per state, a count and nothing else. With the site's
|
||||
//! `needs.yaml` the states that end well are known, and each storyline
|
||||
//! is scored on how many cases it brought in and how many of those
|
||||
//! ended well. Storylines are assigned evenly, so the counts compare
|
||||
//! directly. This is the live half of what the scorer measures offline
|
||||
//! with personas: the same tones, judged by what people did.
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use anyhow::{anyhow, bail, Context};
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Report {
|
||||
storylines: usize,
|
||||
/// bucket -> storyline -> state -> count
|
||||
buckets: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct Needs {
|
||||
#[serde(default)]
|
||||
needs: Vec<Need>,
|
||||
#[serde(default)]
|
||||
tones: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Need {
|
||||
lands_in: String,
|
||||
#[serde(default)]
|
||||
done_when: Vec<String>,
|
||||
#[serde(default)]
|
||||
success: Vec<String>,
|
||||
}
|
||||
|
||||
/// One storyline's showing in one bucket.
|
||||
#[derive(Debug, Default, PartialEq)]
|
||||
struct Showing {
|
||||
arrived: u64,
|
||||
ended_well: u64,
|
||||
}
|
||||
|
||||
/// Per bucket, per storyline index, how many arrived and how many
|
||||
/// ended in a state the need calls success (or, without `success`,
|
||||
/// any finished state).
|
||||
fn showings(counts: &BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>>, needs: &[Need]) -> BTreeMap<String, Vec<Showing>> {
|
||||
let mut out = BTreeMap::new();
|
||||
for (bucket, per_story) in counts {
|
||||
let well: Vec<&str> = needs
|
||||
.iter()
|
||||
.filter(|n| &n.lands_in == bucket)
|
||||
.flat_map(|n| if n.success.is_empty() { n.done_when.iter() } else { n.success.iter() })
|
||||
.map(String::as_str)
|
||||
.collect();
|
||||
let arity = per_story.keys().filter_map(|k| k.parse::<usize>().ok()).max().map(|m| m + 1).unwrap_or(1);
|
||||
let mut rows: Vec<Showing> = (0..arity).map(|_| Showing::default()).collect();
|
||||
for (story, states) in per_story {
|
||||
let Ok(n) = story.parse::<usize>() else { continue };
|
||||
for (state, count) in states {
|
||||
rows[n].arrived += count;
|
||||
if well.contains(&state.as_str()) {
|
||||
rows[n].ended_well += count;
|
||||
}
|
||||
}
|
||||
}
|
||||
out.insert(bucket.clone(), rows);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub fn run(argv: &[String]) -> anyhow::Result<()> {
|
||||
let value = |flag: &str| argv.iter().position(|a| a == flag).and_then(|i| argv.get(i + 1)).cloned();
|
||||
let site = value("--site").ok_or_else(|| anyhow!("usage: portal-trainer report --site <https://the.site> [--needs <needs.yaml>]"))?;
|
||||
let url = format!("{}/report", site.trim_end_matches('/'));
|
||||
let fetched = std::process::Command::new("curl").args(["-sSf", "--max-time", "30", &url]).output().context("running curl")?;
|
||||
if !fetched.status.success() {
|
||||
bail!("could not read {url}: {}", String::from_utf8_lossy(&fetched.stderr).trim());
|
||||
}
|
||||
let report: Report = serde_json::from_slice(&fetched.stdout).with_context(|| format!("reading {url}"))?;
|
||||
let needs: Needs = match value("--needs") {
|
||||
Some(path) => serde_yaml::from_str(&std::fs::read_to_string(&path).with_context(|| format!("reading {path}"))?)?,
|
||||
None => Needs::default(),
|
||||
};
|
||||
println!("{site}: {} storyline(s)", report.storylines);
|
||||
for (i, tone) in needs.tones.iter().enumerate() {
|
||||
println!(" [{i}] {tone}");
|
||||
}
|
||||
if report.buckets.is_empty() {
|
||||
println!("nothing sent in yet");
|
||||
return Ok(());
|
||||
}
|
||||
let rows = showings(&report.buckets, &needs.needs);
|
||||
for (bucket, per_story) in &rows {
|
||||
println!("\n{bucket}");
|
||||
for (n, showing) in per_story.iter().enumerate() {
|
||||
let rate = if showing.arrived > 0 { format!("{:.0}%", 100.0 * showing.ended_well as f64 / showing.arrived as f64) } else { "-".to_string() };
|
||||
println!(" storyline {n}: {} arrived, {} ended well ({rate})", showing.arrived, showing.ended_well);
|
||||
}
|
||||
if per_story.len() > 1 {
|
||||
let best = per_story.iter().enumerate().max_by_key(|(_, s)| (s.ended_well, s.arrived)).map(|(n, _)| n).unwrap_or(0);
|
||||
println!(" most so far: storyline {best}");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_storyline_is_scored_on_what_arrived_and_what_ended_well() {
|
||||
let counts: BTreeMap<String, BTreeMap<String, BTreeMap<String, u64>>> =
|
||||
serde_json::from_str(r#"{"hazards": {"0": {"open": 3, "fixed": 4}, "1": {"open": 1, "fixed": 9, "dropped": 2}}}"#).unwrap();
|
||||
let needs = vec![Need { lands_in: "hazards".into(), done_when: vec!["fixed".into(), "dropped".into()], success: vec!["fixed".into()] }];
|
||||
let rows = showings(&counts, &needs);
|
||||
assert_eq!(rows["hazards"][0], Showing { arrived: 7, ended_well: 4 });
|
||||
assert_eq!(rows["hazards"][1], Showing { arrived: 12, ended_well: 9 });
|
||||
// Without a success list, any finished state counts.
|
||||
let any = vec![Need { lands_in: "hazards".into(), done_when: vec!["fixed".into(), "dropped".into()], success: vec![] }];
|
||||
assert_eq!(showings(&counts, &any)["hazards"][1].ended_well, 11);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
//! `score`: one command for the whole grading of a built site, which
|
||||
//! every run under `runs/` used to carry as its own `score.sh`. The
|
||||
//! lint (iris) writes the visitor tasks and the simulation model, the
|
||||
//! blind visitor answers them with and without descriptions, the lint
|
||||
//! scores the answers, and `rate` reads the cards. What comes out is
|
||||
//! the two routing numbers, the misroutes, the rating table, and a
|
||||
//! `summary.json` a run's README can quote.
|
||||
|
||||
use anyhow::{anyhow, bail, Context};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
pub fn run(argv: &[String]) -> anyhow::Result<()> {
|
||||
let flag = |name: &str| argv.iter().position(|a| a == name).and_then(|i| argv.get(i + 1)).cloned();
|
||||
let usage = "usage: portal-trainer score --site <content dir> --out <dir> [--iris <path>] [--llm 'cmd']";
|
||||
let site = PathBuf::from(flag("--site").ok_or_else(|| anyhow!(usage))?);
|
||||
let out = PathBuf::from(flag("--out").ok_or_else(|| anyhow!(usage))?);
|
||||
let iris = flag("--iris").unwrap_or_else(|| "iris".to_string());
|
||||
let cmd = flag("--llm").unwrap_or_else(|| "claude -p".to_string());
|
||||
let questions = site.join("questions");
|
||||
if !questions.is_dir() {
|
||||
bail!("{} has no questions/ directory", site.display());
|
||||
}
|
||||
std::fs::create_dir_all(&out)?;
|
||||
|
||||
let tasks_full = out.join("tasks-full.jsonl");
|
||||
let tasks_skim = out.join("tasks-skim.jsonl");
|
||||
let model = out.join("model.json");
|
||||
let mut log = String::new();
|
||||
log.push_str(&lint(&iris, &questions, &["--needs-tasks", &s(&tasks_full), "--needs-sim", &s(&model)])?);
|
||||
log.push_str(&lint(&iris, &questions, &["--needs-tasks", &s(&tasks_skim), "--skim"])?);
|
||||
std::fs::write(out.join("iris.log"), &log)?;
|
||||
|
||||
let mut summary = serde_json::Map::new();
|
||||
for (mode, tasks, skim) in [("full", &tasks_full, false), ("skim", &tasks_skim, true)] {
|
||||
eprintln!("score: the visitor answers {mode}");
|
||||
let answers = crate::trainer::answer_blind(&cmd, &std::fs::read_to_string(tasks)?, Some(&site), 0)?;
|
||||
let answers_path = out.join(format!("answers-{mode}.jsonl"));
|
||||
std::fs::write(&answers_path, &answers)?;
|
||||
let (tasks_arg, answers_arg) = (s(tasks), s(&answers_path));
|
||||
let mut args = vec!["--needs-tasks", tasks_arg.as_str(), "--needs-score", answers_arg.as_str()];
|
||||
if skim {
|
||||
args.push("--skim");
|
||||
}
|
||||
let report = lint(&iris, &questions, &args)?;
|
||||
println!("== {}", if skim { "headings only" } else { "with descriptions" });
|
||||
let mut right = None;
|
||||
for line in report.lines() {
|
||||
if line.starts_with("MISROUTE") || line.starts_with("HESITANT") || line.starts_with("UNANSWERED") {
|
||||
println!("{line}");
|
||||
}
|
||||
if let Some(rest) = line.strip_prefix("wording: ") {
|
||||
// "wording: 19/27 routed right (70%)"
|
||||
let mut parts = rest.split_whitespace().next().unwrap_or_default().split('/');
|
||||
let a: usize = parts.next().and_then(|x| x.parse().ok()).unwrap_or(0);
|
||||
let b: usize = parts.next().and_then(|x| x.parse().ok()).unwrap_or(0);
|
||||
right = Some((a, b));
|
||||
println!("{line}");
|
||||
}
|
||||
}
|
||||
let (a, b) = right.ok_or_else(|| anyhow!("iris printed no wording score for {mode}; its output:\n{report}"))?;
|
||||
summary.insert(mode.to_string(), serde_json::json!({ "right": a, "of": b }));
|
||||
}
|
||||
|
||||
println!("== rate");
|
||||
let rated = out.join("rated");
|
||||
let rate_argv: Vec<String> = ["rate", "--model", &s(&model), "--out", &s(&rated), "--repo", &s(&site), "--llm", &cmd].iter().map(|x| x.to_string()).collect();
|
||||
crate::trainer::rate::rate(&rate_argv)?;
|
||||
|
||||
summary.insert("site".into(), serde_json::json!(s(&site)));
|
||||
summary.insert("visitor".into(), serde_json::json!(cmd));
|
||||
std::fs::write(out.join("summary.json"), serde_json::to_string_pretty(&serde_json::Value::Object(summary.clone()))?)?;
|
||||
println!(
|
||||
"== summary: with descriptions {} of {}, headings only {} of {} (rate.json in {})",
|
||||
summary["full"]["right"], summary["full"]["of"], summary["skim"]["right"], summary["skim"]["of"], rated.display()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn s(path: &Path) -> String {
|
||||
path.display().to_string()
|
||||
}
|
||||
|
||||
/// `iris check --path <questions> <args>`; stdout and stderr together,
|
||||
/// since the lint reports on both. A lint that fails is a site that
|
||||
/// cannot be scored, and its output is the reason.
|
||||
fn lint(iris: &str, questions: &Path, args: &[&str]) -> anyhow::Result<String> {
|
||||
let output = Command::new(iris)
|
||||
.arg("check")
|
||||
.arg("--path")
|
||||
.arg(questions)
|
||||
.args(args)
|
||||
.output()
|
||||
.with_context(|| format!("running {iris} (pass --iris <path> if it is not on PATH)"))?;
|
||||
let text = format!("{}{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr));
|
||||
if !output.status.success() {
|
||||
bail!("{iris} check --path {} {} failed:\n{text}", questions.display(), args.join(" "));
|
||||
}
|
||||
Ok(text)
|
||||
}
|
||||
@@ -0,0 +1,678 @@
|
||||
//! Plays the whole business, many times.
|
||||
//!
|
||||
//! Input is the model `question_lint --needs-sim` exports. Each case is
|
||||
//! one persona on one run. A visitor model picks an alternative and
|
||||
//! fills the form in that person's words, in a manner that varies by
|
||||
//! run; the record lands in its bucket. Then the case is walked along
|
||||
//! the handling group's own desk buttons to a finished state - a
|
||||
//! different one each run, so every state and button gets exercised -
|
||||
//! and at each button a handler model is asked the one thing a model
|
||||
//! can judge: with only what the form collected, could this role
|
||||
//! really do what the button claims? If not, its question goes back to
|
||||
//! the visitor, who may or may not know the answer.
|
||||
//!
|
||||
//! What comes out is about the content, never the market:
|
||||
//! - a **misroute** is wording;
|
||||
//! - an **abandoned** form asks too much of that person;
|
||||
//! - an **ask-back** at (state, button) means the form did not collect
|
||||
//! what the role needs there, and the question names the field;
|
||||
//! - an **unanswerable** ask-back is a fact the visitor never had, so
|
||||
//! the process itself has to supply it;
|
||||
//! - **no route** means a finished state the desk cannot reach.
|
||||
//!
|
||||
//! Who wins a job is not simulated. That comes from the live bucket:
|
||||
//! the `success` states in needs.yaml are what a report over real
|
||||
//! records counts.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::io::Write as _;
|
||||
|
||||
use anyhow::{anyhow, Context};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::trainer::llm_with;
|
||||
|
||||
const TRAITS: &[&str] = &[
|
||||
"is precise and businesslike",
|
||||
"types two short lines on a phone and skips anything optional",
|
||||
"explains at length, with every detail they can think of",
|
||||
"is unsure of the right terms and describes things loosely",
|
||||
"is in a hurry and a little impatient",
|
||||
];
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
|
||||
#[serde(tag = "outcome", content = "detail", rename_all = "snake_case")]
|
||||
pub enum Outcome {
|
||||
Finished(String),
|
||||
Stuck(String),
|
||||
Misrouted(String),
|
||||
Abandoned(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct AskBack {
|
||||
pub state: String,
|
||||
pub button: String,
|
||||
pub question: String,
|
||||
/// The visitor knew the answer. When false, no form field would
|
||||
/// have helped: the process has to find it out another way.
|
||||
pub answerable: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct Case {
|
||||
persona: String,
|
||||
need: String,
|
||||
run: usize,
|
||||
target: String,
|
||||
record: BTreeMap<String, Value>,
|
||||
states: Vec<String>,
|
||||
/// "state -> button", in the order pressed.
|
||||
pressed: Vec<String>,
|
||||
/// Who pressed each of them: a group, or `submitter`.
|
||||
moved_by: Vec<String>,
|
||||
touches: usize,
|
||||
ask_backs: Vec<AskBack>,
|
||||
outcome: Outcome,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct NeedScore {
|
||||
pub cases: usize,
|
||||
pub finished: usize,
|
||||
pub stuck: usize,
|
||||
pub misrouted: usize,
|
||||
pub abandoned: usize,
|
||||
pub touches: usize,
|
||||
pub ask_backs: usize,
|
||||
pub unanswerable: usize,
|
||||
pub asked: Vec<AskBack>,
|
||||
/// "state -> button": (times pressed, times the role had to ask first)
|
||||
pub buttons: BTreeMap<String, (usize, usize)>,
|
||||
pub stuck_at: Vec<String>,
|
||||
pub misroutes: Vec<String>,
|
||||
pub abandon_reasons: Vec<String>,
|
||||
}
|
||||
|
||||
pub fn run(argv: &[String]) -> anyhow::Result<()> {
|
||||
let value = |flag: &str| {
|
||||
argv.iter()
|
||||
.position(|a| a == flag)
|
||||
.and_then(|i| argv.get(i + 1))
|
||||
.cloned()
|
||||
};
|
||||
let usage = "usage: portal-trainer simulate --model <model.json> --out <dir> [--llm 'cmd'] [--handler-llm 'cmd'] [--runs 3] [--need <id>]";
|
||||
let model_path = value("--model").ok_or_else(|| anyhow!(usage))?;
|
||||
let out = std::path::PathBuf::from(value("--out").ok_or_else(|| anyhow!(usage))?);
|
||||
let visitor_cmd = value("--llm").unwrap_or_else(|| "claude -p".to_string());
|
||||
let handler_cmd = value("--handler-llm").unwrap_or_else(|| visitor_cmd.clone());
|
||||
let runs: usize = value("--runs").map(|v| v.parse()).transpose()?.unwrap_or(3);
|
||||
let only = value("--need");
|
||||
|
||||
let model: Value = serde_json::from_str(
|
||||
&std::fs::read_to_string(&model_path).with_context(|| format!("reading {model_path}"))?,
|
||||
)?;
|
||||
std::fs::create_dir_all(&out)?;
|
||||
let mut cases_file = std::fs::File::create(out.join("cases.jsonl"))?;
|
||||
let mut scores: BTreeMap<String, NeedScore> = BTreeMap::new();
|
||||
|
||||
// A dry run asks no model anything: it walks every route of every
|
||||
// aggregate, so each button of each role is pressed once. Its cases
|
||||
// are what `needs_replay` puts through the real engine.
|
||||
if argv.iter().any(|a| a == "--dry") {
|
||||
for need in model["needs"].as_array().into_iter().flatten() {
|
||||
let id = need["id"].as_str().unwrap_or_default();
|
||||
if only.as_deref().is_some_and(|o| o != id) {
|
||||
continue;
|
||||
}
|
||||
let bucket = &model["buckets"][need["lands_in"].as_str().unwrap_or_default()];
|
||||
let done: Vec<&str> = need["done_when"].as_array().into_iter().flatten().filter_map(Value::as_str).collect();
|
||||
for (run, route) in all_routes(bucket, &done).into_iter().enumerate() {
|
||||
let mut states = vec![bucket["initial"].as_str().unwrap_or("open").to_string()];
|
||||
states.extend(route.iter().map(|m| m.to.clone()));
|
||||
let case = Case {
|
||||
persona: "route".to_string(),
|
||||
need: id.to_string(),
|
||||
run,
|
||||
target: states.last().cloned().unwrap_or_default(),
|
||||
record: BTreeMap::from([("note".to_string(), json!("dry run"))]),
|
||||
pressed: route.iter().map(|m| format!("{} -> {}", m.from, m.label)).collect(),
|
||||
moved_by: route.iter().map(|m| m.by.clone()).collect(),
|
||||
touches: route.len(),
|
||||
ask_backs: Vec::new(),
|
||||
outcome: Outcome::Finished(states.last().cloned().unwrap_or_default()),
|
||||
states,
|
||||
};
|
||||
tally(scores.entry(id.to_string()).or_default(), &case);
|
||||
writeln!(cases_file, "{}", serde_json::to_string(&case)?)?;
|
||||
}
|
||||
}
|
||||
std::fs::write(out.join("score.json"), serde_json::to_string_pretty(&scores)?)?;
|
||||
let relays: usize = scores.values().map(|s| s.cases).sum();
|
||||
println!("dry run: {relays} route(s) across {} need(s) written to {}", scores.len(), out.join("cases.jsonl").display());
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for persona in model["personas"].as_array().into_iter().flatten() {
|
||||
let need_id = persona["need"].as_str().unwrap_or_default();
|
||||
if only.as_deref().is_some_and(|o| o != need_id) {
|
||||
continue;
|
||||
}
|
||||
let Some(need) = model["needs"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.find(|n| n["id"] == need_id)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
for run in 0..runs {
|
||||
let case = play(&model, need, persona, run, &visitor_cmd, &handler_cmd)?;
|
||||
eprintln!(
|
||||
"{} #{run}: {:?} after {} touch(es), {} ask-back(s)",
|
||||
case.persona,
|
||||
case.outcome,
|
||||
case.touches,
|
||||
case.ask_backs.len()
|
||||
);
|
||||
tally(scores.entry(need_id.to_string()).or_default(), &case);
|
||||
writeln!(cases_file, "{}", serde_json::to_string(&case)?)?;
|
||||
}
|
||||
}
|
||||
std::fs::write(out.join("score.json"), serde_json::to_string_pretty(&scores)?)?;
|
||||
print_scores(&scores);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn play(
|
||||
model: &Value,
|
||||
need: &Value,
|
||||
persona: &Value,
|
||||
run: usize,
|
||||
visitor_cmd: &str,
|
||||
handler_cmd: &str,
|
||||
) -> anyhow::Result<Case> {
|
||||
let who = persona["text"].as_str().unwrap_or_default();
|
||||
let manner = TRAITS[run % TRAITS.len()];
|
||||
let seed = |step: usize| (run * 1000 + step).to_string();
|
||||
let mut case = Case {
|
||||
persona: persona["id"].as_str().unwrap_or_default().to_string(),
|
||||
need: need["id"].as_str().unwrap_or_default().to_string(),
|
||||
run,
|
||||
target: String::new(),
|
||||
record: BTreeMap::new(),
|
||||
states: Vec::new(),
|
||||
pressed: Vec::new(),
|
||||
moved_by: Vec::new(),
|
||||
touches: 0,
|
||||
ask_backs: Vec::new(),
|
||||
outcome: Outcome::Stuck("never started".to_string()),
|
||||
};
|
||||
|
||||
// The visitor's side.
|
||||
for (i, step) in need["steps"].as_array().into_iter().flatten().enumerate() {
|
||||
let options = step["options"].as_array().cloned().unwrap_or_default();
|
||||
if options.len() > 1 {
|
||||
let prompt = format!(
|
||||
"You are this person: {who}\n\nYou open a web page that asks: {}\nIt offers:\n{}\nWhich one do you pick? Judge only from your own situation and the wording.\nReply as JSON: {{\"choice\": \"<option heading, copied exactly>\"}}",
|
||||
step["question"],
|
||||
options
|
||||
.iter()
|
||||
.map(|o| format!("- {}: {}", o["name"].as_str().unwrap_or_default(), o["description"].as_str().unwrap_or_default()))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
);
|
||||
let reply = ask(visitor_cmd, &prompt, &seed(i * 10))?;
|
||||
let names: Vec<&str> = options.iter().filter_map(|o| o["name"].as_str()).collect();
|
||||
let choice = resolve_choice(&names, reply["choice"].as_str().unwrap_or_default());
|
||||
let expected = step["expect"].as_array().cloned().unwrap_or_default();
|
||||
if !expected.iter().any(|e| e.as_str() == Some(choice.as_str())) {
|
||||
case.outcome = Outcome::Misrouted(choice);
|
||||
return Ok(case);
|
||||
}
|
||||
}
|
||||
let fields = step["fields"].as_array().cloned().unwrap_or_default();
|
||||
if fields.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let prompt = format!(
|
||||
"You are this person: {who}\nToday you are someone who {manner}.\n\nYou chose \"{}\" and now face this form:\n{}\n\nFill it in as yourself. Use only what you would actually know or have at hand; never invent precise facts a person like you would not know - say so in the field instead. Optional fields may be left out. If this form asks for something you would refuse to give, or is more effort than your situation is worth to you, abandon it.\nReply as JSON: {{\"abandon\": false, \"why\": \"\", \"values\": {{\"<field name>\": \"<what you type>\"}}}}",
|
||||
step["alternative"].as_str().unwrap_or_default(),
|
||||
fields
|
||||
.iter()
|
||||
.map(|f| format!(
|
||||
"- {} (name: {}, type: {}{}{})",
|
||||
f["label"].as_str().unwrap_or_default(),
|
||||
f["name"].as_str().unwrap_or_default(),
|
||||
f["type"].as_str().unwrap_or("text"),
|
||||
if f["optional"] == true { ", optional" } else { ", required" },
|
||||
match f["options"].as_array() {
|
||||
Some(o) if !o.is_empty() => format!(", one of {}", Value::Array(o.clone())),
|
||||
_ => String::new(),
|
||||
}
|
||||
))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
);
|
||||
let reply = ask(visitor_cmd, &prompt, &seed(i * 10 + 1))?;
|
||||
if reply["abandon"] == true {
|
||||
case.outcome = Outcome::Abandoned(reply["why"].as_str().unwrap_or_default().to_string());
|
||||
return Ok(case);
|
||||
}
|
||||
if let Some(values) = reply["values"].as_object() {
|
||||
for (k, v) in values {
|
||||
case.record.insert(k.clone(), v.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The business's side. A model is a poor judge of who wins a job,
|
||||
// and an 8B one decides by the tone of its prompt (measured: every
|
||||
// case "won", then, one sentence later, every case "lost"). So the
|
||||
// route is scripted, not decided: run by run, cases are sent to
|
||||
// each finished state in turn, along the desk's own buttons, which
|
||||
// covers every state and button of the machine. At each button the
|
||||
// model is asked only what it can judge: with what the form
|
||||
// collected, could this role really do what the button claims?
|
||||
let bucket_name = need["lands_in"].as_str().unwrap_or_default();
|
||||
let bucket = &model["buckets"][bucket_name];
|
||||
let done: Vec<&str> = need["done_when"].as_array().into_iter().flatten().filter_map(Value::as_str).collect();
|
||||
let initial = bucket["initial"].as_str().unwrap_or("open").to_string();
|
||||
let target = done.get(run % done.len().max(1)).copied().unwrap_or_default().to_string();
|
||||
case.target = target.clone();
|
||||
case.states.push(initial.clone());
|
||||
let Some(route) = route_to(bucket, &initial, &target) else {
|
||||
case.outcome = Outcome::Stuck(format!("no route of buttons from {initial} to {target}"));
|
||||
return Ok(case);
|
||||
};
|
||||
for (step, mv) in route.iter().enumerate() {
|
||||
let (from, label, to, group) = (&mv.from, &mv.label, &mv.to, &mv.by);
|
||||
case.moved_by.push(group.clone());
|
||||
case.touches += 1;
|
||||
case.pressed.push(format!("{from} -> {label}"));
|
||||
case.states.push(to.clone());
|
||||
// The submitter's own decision is theirs to make; there is no
|
||||
// role whose information could be short.
|
||||
if group == SUBMITTER {
|
||||
continue;
|
||||
}
|
||||
let prompt = format!(
|
||||
"You work at a business as one of its {group}. A case in `{bucket_name}` is in state `{from}`. It came from {}, who wants {}.\n\nThis is everything you have about it - what the form collected, plus any follow-up:\n{}\n\nThe next thing that happens to this case is the desk button \"{label}\", which moves it to `{to}`. Pressing it claims that this has really been done.\n\nUsing only what is in front of you, could you really do that now? Answer false only when a fact that ONLY THE PERSON can give is missing and you could not proceed without it. Facts you find out yourself while doing the work do not count: you learn a building's condition by visiting it, a firm's registration by looking it up.\nReply as JSON: {{\"can_act\": true or false, \"need_to_know\": \"<the one question you would send them>\" or null}}",
|
||||
need["who"].as_str().unwrap_or_default(),
|
||||
need["wants"].as_str().unwrap_or_default(),
|
||||
serde_json::to_string_pretty(&case.record)?,
|
||||
);
|
||||
let reply = ask(handler_cmd, &prompt, &seed(100 + step))?;
|
||||
let question = reply["need_to_know"].as_str().unwrap_or_default().trim().to_string();
|
||||
if reply["can_act"] == false && !question.is_empty() {
|
||||
let answer = ask(
|
||||
visitor_cmd,
|
||||
&format!(
|
||||
"You are this person: {who}\nToday you are someone who {manner}.\n\nThe business you wrote to comes back with a question: {question}\nAnswer as yourself, briefly. Use only what a person like you would really know.\nReply as JSON: {{\"known\": true or false, \"answer\": \"...\"}}"
|
||||
),
|
||||
&seed(200 + step),
|
||||
)?;
|
||||
let n = case.ask_backs.len() + 1;
|
||||
case.record.insert(format!("follow_up_{n}"), json!({ "asked": question, "answered": answer["answer"] }));
|
||||
case.ask_backs.push(AskBack {
|
||||
state: from.clone(),
|
||||
button: label.clone(),
|
||||
question,
|
||||
answerable: answer["known"] != false,
|
||||
});
|
||||
case.touches += 1;
|
||||
}
|
||||
}
|
||||
case.outcome = Outcome::Finished(target);
|
||||
Ok(case)
|
||||
}
|
||||
|
||||
pub(crate) const SUBMITTER: &str = "submitter";
|
||||
|
||||
/// One button: who holds it, and where it takes a case.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(crate) struct Move {
|
||||
pub from: String,
|
||||
pub label: String,
|
||||
pub to: String,
|
||||
pub by: String,
|
||||
}
|
||||
|
||||
/// Every legal button on an aggregate, whoever holds it: each group's
|
||||
/// desk, and the submitter's own link. Roles overlap on one aggregate,
|
||||
/// so a route is a relay, not one desk's work.
|
||||
pub(crate) fn all_moves(bucket: &Value) -> Vec<Move> {
|
||||
let legal = |from: &str, to: &str| bucket["transitions"][from].as_array().is_some_and(|a| a.iter().any(|x| x == to));
|
||||
let mut moves: Vec<Move> = Vec::new();
|
||||
for desk in bucket["desks"].as_array().into_iter().flatten() {
|
||||
for m in desk["moves"].as_array().into_iter().flatten() {
|
||||
let (Some(from), Some(label), Some(to)) = (m["from"].as_str(), m["label"].as_str(), m["to"].as_str()) else { continue };
|
||||
if legal(from, to) && !moves.iter().any(|x| x.from == from && x.to == to) {
|
||||
moves.push(Move { from: from.into(), label: label.into(), to: to.into(), by: desk["group"].as_str().unwrap_or("ungated").into() });
|
||||
}
|
||||
}
|
||||
}
|
||||
let states: Vec<String> = bucket["transitions"].as_object().map(|t| t.keys().cloned().collect()).unwrap_or_default();
|
||||
for to in bucket["self_moves"].as_array().into_iter().flatten().filter_map(Value::as_str) {
|
||||
for from in &states {
|
||||
if legal(from, to) && !moves.iter().any(|x| &x.from == from && x.to == to) {
|
||||
moves.push(Move { from: from.clone(), label: "their own link".into(), to: to.into(), by: SUBMITTER.into() });
|
||||
}
|
||||
}
|
||||
}
|
||||
moves
|
||||
}
|
||||
|
||||
/// The shortest relay of buttons from one state to another.
|
||||
fn route_to(bucket: &Value, from: &str, to: &str) -> Option<Vec<Move>> {
|
||||
let moves = all_moves(bucket);
|
||||
let mut queue = std::collections::VecDeque::from([(from.to_string(), Vec::new())]);
|
||||
let mut seen = std::collections::HashSet::from([from.to_string()]);
|
||||
while let Some((state, path)) = queue.pop_front() {
|
||||
if state == to {
|
||||
return Some(path);
|
||||
}
|
||||
for mv in moves.iter().filter(|m| m.from == state) {
|
||||
if seen.insert(mv.to.clone()) {
|
||||
let mut next = path.clone();
|
||||
next.push(mv.clone());
|
||||
queue.push_back((mv.to.clone(), next));
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Every simple route from the first state to any finished one: what a
|
||||
/// dry run walks, so that each button of each role is pressed at least
|
||||
/// once against the real engine.
|
||||
pub(crate) fn all_routes(bucket: &Value, done: &[&str]) -> Vec<Vec<Move>> {
|
||||
fn walk(state: &str, done: &[&str], moves: &[Move], path: &mut Vec<Move>, out: &mut Vec<Vec<Move>>) {
|
||||
if done.contains(&state) {
|
||||
out.push(path.clone());
|
||||
return;
|
||||
}
|
||||
for mv in moves.iter().filter(|m| m.from == state) {
|
||||
if path.iter().any(|p| p.from == mv.to) {
|
||||
continue;
|
||||
}
|
||||
path.push(mv.clone());
|
||||
walk(&mv.to, done, moves, path, out);
|
||||
path.pop();
|
||||
}
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
walk(bucket["initial"].as_str().unwrap_or("open"), done, &all_moves(bucket), &mut Vec::new(), &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// Models often answer "Heading: its description" or change the case.
|
||||
/// The pick is the longest option heading the answer starts with;
|
||||
/// anything else is kept verbatim and counts as a misroute.
|
||||
pub(crate) fn resolve_choice(names: &[&str], raw: &str) -> String {
|
||||
let answer = raw.trim().trim_matches('"').to_lowercase();
|
||||
names
|
||||
.iter()
|
||||
.filter(|n| answer.starts_with(&n.to_lowercase()))
|
||||
.max_by_key(|n| n.len())
|
||||
.map(|n| n.to_string())
|
||||
.unwrap_or_else(|| raw.trim().to_string())
|
||||
}
|
||||
|
||||
/// One JSON-answering call. The backend is asked for JSON where it can
|
||||
/// be (`FORMAT=json`, honoured by engines/ollama.sh) and the first
|
||||
/// object in the reply is taken either way.
|
||||
pub(crate) fn ask(cmd: &str, prompt: &str, seed: &str) -> anyhow::Result<Value> {
|
||||
let reply = llm_with(cmd, prompt, &[("FORMAT", "json"), ("SEED", seed), ("TEMPERATURE", "0.7")])?;
|
||||
Ok(first_object(&reply).unwrap_or(Value::Null))
|
||||
}
|
||||
|
||||
pub(crate) fn first_object(text: &str) -> Option<Value> {
|
||||
let start = text.find('{')?;
|
||||
let mut depth = 0usize;
|
||||
let mut in_string = false;
|
||||
let mut escaped = false;
|
||||
for (i, c) in text[start..].char_indices() {
|
||||
match (in_string, escaped, c) {
|
||||
(true, true, _) => escaped = false,
|
||||
(true, false, '\\') => escaped = true,
|
||||
(true, false, '"') => in_string = false,
|
||||
(false, _, '"') => in_string = true,
|
||||
(false, _, '{') => depth += 1,
|
||||
(false, _, '}') => {
|
||||
depth -= 1;
|
||||
if depth == 0 {
|
||||
return serde_json::from_str(&text[start..start + i + 1]).ok();
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn tally(score: &mut NeedScore, case: &Case) {
|
||||
score.cases += 1;
|
||||
score.touches += case.touches;
|
||||
score.ask_backs += case.ask_backs.len();
|
||||
score.unanswerable += case.ask_backs.iter().filter(|a| !a.answerable).count();
|
||||
score.asked.extend(case.ask_backs.iter().cloned());
|
||||
for button in &case.pressed {
|
||||
score.buttons.entry(button.clone()).or_default().0 += 1;
|
||||
}
|
||||
for a in &case.ask_backs {
|
||||
score.buttons.entry(format!("{} -> {}", a.state, a.button)).or_default().1 += 1;
|
||||
}
|
||||
match &case.outcome {
|
||||
Outcome::Finished(_) => score.finished += 1,
|
||||
Outcome::Stuck(at) => {
|
||||
score.stuck += 1;
|
||||
score.stuck_at.push(at.clone());
|
||||
}
|
||||
Outcome::Misrouted(picked) => {
|
||||
score.misrouted += 1;
|
||||
score.misroutes.push(format!("{}: {picked}", case.persona));
|
||||
}
|
||||
Outcome::Abandoned(why) => {
|
||||
score.abandoned += 1;
|
||||
score.abandon_reasons.push(format!("{}: {why}", case.persona));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn pct(n: usize, of: usize) -> String {
|
||||
if of == 0 {
|
||||
"-".to_string()
|
||||
} else {
|
||||
format!("{:.0}%", n as f64 * 100.0 / of as f64)
|
||||
}
|
||||
}
|
||||
|
||||
fn per_case(n: usize, s: &NeedScore) -> String {
|
||||
let reached = s.cases - s.misrouted - s.abandoned;
|
||||
if reached == 0 {
|
||||
"-".to_string()
|
||||
} else {
|
||||
format!("{:.1}", n as f64 / reached as f64)
|
||||
}
|
||||
}
|
||||
|
||||
fn print_scores(scores: &BTreeMap<String, NeedScore>) {
|
||||
println!(
|
||||
"\n{:<28} {:>5} {:>9} {:>9} {:>10} {:>9} {:>8} {:>10} {:>13}",
|
||||
"need", "cases", "finished", "misroute", "abandoned", "no route", "touches", "ask-backs", "unanswerable"
|
||||
);
|
||||
for (need, s) in scores {
|
||||
println!(
|
||||
"{:<28} {:>5} {:>9} {:>9} {:>10} {:>9} {:>8} {:>10} {:>13}",
|
||||
need,
|
||||
s.cases,
|
||||
pct(s.finished, s.cases),
|
||||
pct(s.misrouted, s.cases),
|
||||
pct(s.abandoned, s.cases),
|
||||
pct(s.stuck, s.cases),
|
||||
per_case(s.touches, s),
|
||||
per_case(s.ask_backs, s),
|
||||
per_case(s.unanswerable, s),
|
||||
);
|
||||
}
|
||||
println!("\nWhere the role could not act on what the form collected:");
|
||||
for (need, s) in scores {
|
||||
for (button, (pressed, asked)) in s.buttons.iter().filter(|(_, (_, asked))| *asked > 0) {
|
||||
println!(" {need}: {button} - had to ask first in {asked} of {pressed}");
|
||||
}
|
||||
}
|
||||
for (need, s) in scores {
|
||||
for a in &s.asked {
|
||||
println!(
|
||||
" {need} [{} / {}]{} {}",
|
||||
a.state,
|
||||
a.button,
|
||||
if a.answerable { "" } else { " (visitor did not know)" },
|
||||
a.question
|
||||
);
|
||||
}
|
||||
for at in &s.stuck_at {
|
||||
println!(" {need}: {at}");
|
||||
}
|
||||
for m in &s.misroutes {
|
||||
println!(" {need}: misrouted - {m}");
|
||||
}
|
||||
for r in &s.abandon_reasons {
|
||||
println!(" {need}: abandoned - {r}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `compare <a/score.json> <b/score.json>`: the same seeded cases
|
||||
/// against two versions of a site.
|
||||
pub fn compare(argv: &[String]) -> anyhow::Result<()> {
|
||||
let (a, b) = match (argv.get(1), argv.get(2)) {
|
||||
(Some(a), Some(b)) => (a, b),
|
||||
_ => return Err(anyhow!("usage: portal-trainer compare <a/score.json> <b/score.json>")),
|
||||
};
|
||||
let load = |p: &String| -> anyhow::Result<BTreeMap<String, NeedScore>> {
|
||||
Ok(serde_json::from_str(&std::fs::read_to_string(p).with_context(|| format!("reading {p}"))?)?)
|
||||
};
|
||||
let (a, b) = (load(a)?, load(b)?);
|
||||
println!(
|
||||
"{:<28} {:>14} {:>14} {:>14} {:>16} {:>16}",
|
||||
"need", "misroute", "abandoned", "no route", "touches / case", "ask-backs / case"
|
||||
);
|
||||
for (need, sa) in &a {
|
||||
let Some(sb) = b.get(need) else { continue };
|
||||
println!(
|
||||
"{:<28} {:>14} {:>14} {:>14} {:>16} {:>16}",
|
||||
need,
|
||||
format!("{} -> {}", pct(sa.misrouted, sa.cases), pct(sb.misrouted, sb.cases)),
|
||||
format!("{} -> {}", pct(sa.abandoned, sa.cases), pct(sb.abandoned, sb.cases)),
|
||||
format!("{} -> {}", pct(sa.stuck, sa.cases), pct(sb.stuck, sb.cases)),
|
||||
format!("{} -> {}", per_case(sa.touches, sa), per_case(sb.touches, sb)),
|
||||
format!("{} -> {}", per_case(sa.ask_backs, sa), per_case(sb.ask_backs, sb)),
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_first_json_object_is_found_inside_chatter() {
|
||||
let v = first_object("Sure! {\"choice\": \"A {b}\", \"n\": {\"x\": 1}} trailing").unwrap();
|
||||
assert_eq!(v["choice"], "A {b}");
|
||||
assert_eq!(v["n"]["x"], 1);
|
||||
assert!(first_object("no json").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_choice_is_matched_by_its_heading_however_the_model_dresses_it() {
|
||||
let names = ["Ask about working", "Ask about working with us", "Get a price"];
|
||||
assert_eq!(resolve_choice(&names, "Ask about working with us: For carpenters who..."), "Ask about working with us");
|
||||
assert_eq!(resolve_choice(&names, " get a PRICE "), "Get a price");
|
||||
assert_eq!(resolve_choice(&names, "Something else"), "Something else");
|
||||
}
|
||||
|
||||
fn bucket() -> Value {
|
||||
json!({
|
||||
"initial": "open",
|
||||
"transitions": {"open": ["surveyed", "lost"], "surveyed": ["quoted", "lost"], "quoted": ["won", "lost"]},
|
||||
"desks": [
|
||||
{"group": "pm", "moves": [
|
||||
{"from": "open", "to": "surveyed", "label": "Looked"},
|
||||
{"from": "surveyed", "to": "quoted", "label": "Quote sent"},
|
||||
{"from": "quoted", "to": "won", "label": "Won"},
|
||||
{"from": "open", "to": "lost", "label": "Not for us"},
|
||||
// Offered by the desk, refused by the machine.
|
||||
{"from": "open", "to": "won", "label": "Shortcut"}
|
||||
]},
|
||||
{"group": "other", "moves": [{"from": "quoted", "to": "lost", "label": "Theirs"}]}
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_route_is_a_relay_of_legal_buttons_across_roles() {
|
||||
// "Shortcut" is offered by a desk and refused by the machine.
|
||||
assert!(!all_moves(&bucket()).iter().any(|m| m.label == "Shortcut"));
|
||||
// Losing a quoted job is the other group's button, so the way
|
||||
// from open to that ending is a relay: two roles, one case.
|
||||
let route = route_to(&bucket(), "surveyed", "lost").unwrap();
|
||||
let relay: Vec<(&str, &str)> = route.iter().map(|m| (m.label.as_str(), m.by.as_str())).collect();
|
||||
assert_eq!(relay, [("Quote sent", "pm"), ("Theirs", "other")]);
|
||||
assert_eq!(route_to(&bucket(), "open", "open").unwrap().len(), 0);
|
||||
assert!(route_to(&bucket(), "won", "open").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_submitters_own_link_is_a_move_like_any_other() {
|
||||
let mut b = bucket();
|
||||
b["self_moves"] = json!(["lost"]);
|
||||
let theirs: Vec<Move> = all_moves(&b).into_iter().filter(|m| m.by == SUBMITTER).collect();
|
||||
// Only where the machine allows it and no desk already holds it:
|
||||
// open->lost is the project manager's and quoted->lost the other
|
||||
// group's, which leaves surveyed->lost to the link.
|
||||
let froms: Vec<&str> = theirs.iter().map(|m| m.from.as_str()).collect();
|
||||
assert_eq!(froms, ["surveyed"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_dry_run_walks_every_route_to_every_ending() {
|
||||
let routes = all_routes(&bucket(), &["won", "lost"]);
|
||||
let ends: Vec<String> = routes.iter().map(|r| format!("{}:{}", r.len(), r.last().unwrap().to)).collect();
|
||||
// open->lost; open->surveyed->quoted->won; and the relay
|
||||
// open->surveyed->quoted->lost, closed by the other group.
|
||||
assert_eq!(ends.len(), 3, "{ends:?}");
|
||||
assert_eq!(ends.iter().filter(|e| *e == "3:lost").count(), 1);
|
||||
assert!(ends.contains(&"1:lost".to_string()) && ends.contains(&"3:won".to_string()));
|
||||
assert!(routes.iter().any(|r| r.iter().any(|m| m.by == "other")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_case_is_tallied_by_how_it_ended_and_what_had_to_be_asked() {
|
||||
let mut score = NeedScore::default();
|
||||
let base = |outcome| Case {
|
||||
persona: "p".into(),
|
||||
need: "n".into(),
|
||||
run: 0,
|
||||
target: "won".into(),
|
||||
record: BTreeMap::new(),
|
||||
states: vec![],
|
||||
pressed: vec!["open -> Looked".into(), "surveyed -> Quote sent".into()],
|
||||
moved_by: vec!["pm".into(), "pm".into()],
|
||||
touches: 3,
|
||||
ask_backs: vec![
|
||||
AskBack { state: "open".into(), button: "Looked".into(), question: "Which floor?".into(), answerable: true },
|
||||
AskBack { state: "surveyed".into(), button: "Quote sent".into(), question: "Load-bearing?".into(), answerable: false },
|
||||
],
|
||||
outcome,
|
||||
};
|
||||
tally(&mut score, &base(Outcome::Finished("won".into())));
|
||||
tally(&mut score, &base(Outcome::Stuck("no route".into())));
|
||||
tally(&mut score, &base(Outcome::Misrouted("Wrong door".into())));
|
||||
assert_eq!((score.cases, score.finished, score.stuck, score.misrouted), (3, 1, 1, 1));
|
||||
assert_eq!((score.ask_backs, score.unanswerable), (6, 3));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user