diff --git a/README.md b/README.md index c9fb6b7..e9b017c 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,17 @@ stream. too: a `followup` page is offered only when it is on that path (the page the chain verifiably answered) or further down from the current page over `action` edges - one followup path never unlocks another. +- **Business needs as a checkable spec** (`src/needs.rs`, + `question_lint`): a content repo may ship a `needs.yaml` saying who + wants what, which bucket the answer must land in, and which group + has to carry it to a finished state. The lint proves the structure + (a path of submissions within an effort budget, a desk held by that + group, every finished state reachable, no record stranding; a + `planned` need warns instead of failing). Its personas - visitors in + their own words - become typed `choice` tasks for any external + answering engine (`--needs-tasks`, `--skim` for headings only) whose + picks are graded with `--needs-score`: a misroute is a wording + defect. The runtime never reads the file. - **Live resources** (`src/resource.rs`): content can pull a KV bucket, Gitea starred/org repos/releases, or any public JSON URL (SSRF fail-closed), reshaped by a content-declared `jq` filter. diff --git a/src/bin/question_lint.rs b/src/bin/question_lint.rs index feb46bb..20ad00a 100644 --- a/src/bin/question_lint.rs +++ b/src/bin/question_lint.rs @@ -9,7 +9,7 @@ //! static binary," not "build a Rust workspace." #![cfg(feature = "ssr")] -use portal::{aggregates, content}; +use portal::{aggregates, content, needs}; #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -18,6 +18,10 @@ async fn main() -> anyhow::Result<()> { let mut branch = "main".to_string(); let mut subdir = "questions".to_string(); let mut path: Option = None; + let mut tasks_out: Option = None; + let mut answers_in: Option = None; + let mut min_accuracy: f64 = 0.0; + let mut skim = false; while let Some(arg) = args.next() { match arg.as_str() { @@ -25,6 +29,12 @@ async fn main() -> anyhow::Result<()> { "--branch" => branch = args.next().unwrap_or(branch), "--subdir" => subdir = args.next().unwrap_or(subdir), "--path" => path = args.next(), + "--needs-tasks" => tasks_out = args.next(), + "--skim" => skim = true, + "--needs-score" => answers_in = args.next(), + "--min-accuracy" => { + min_accuracy = args.next().and_then(|v| v.parse().ok()).unwrap_or(min_accuracy) + } other => { eprintln!("unknown argument: {other}"); std::process::exit(2); @@ -32,18 +42,24 @@ async fn main() -> anyhow::Result<()> { } } - let (questions, aggregates_map, site) = if let Some(dir) = path { - (load_from_dir(&dir)?, load_aggregates_from_dir(&dir)?, load_site_from_dir(&dir)?) + let (questions, aggregates_map, site, needs_raw) = if let Some(dir) = path { + ( + load_from_dir(&dir)?, + load_aggregates_from_dir(&dir)?, + load_site_from_dir(&dir)?, + load_needs_from_dir(&dir)?, + ) } else if let Some(repo_url) = repo { let questions = content::load_questions_from_gitea(&repo_url, &branch, &subdir).await?; let aggregates_map = content::load_aggregates_from_gitea(&repo_url, &branch).await?; // load_site_from_gitea already validates; a missing file is // the default config, same as at portal boot. let site = content::load_site_from_gitea(&repo_url, &branch).await?; - (questions, aggregates_map, site) + let needs_raw = content::load_needs_from_gitea(&repo_url, &branch).await?; + (questions, aggregates_map, site, needs_raw) } else { eprintln!( - "usage: question-lint --repo [--branch main] [--subdir questions] | --path " + "usage: question-lint (--repo [--branch main] [--subdir questions] | --path ) [--needs-tasks [--skim]] [--needs-score [--min-accuracy 0.8]]" ); std::process::exit(2); }; @@ -72,7 +88,15 @@ async fn main() -> anyhow::Result<()> { questions.len(), aggregates_map.len() ); - Ok(()) + check_needs( + needs_raw.as_deref(), + &questions, + &aggregates_map, + tasks_out.as_deref(), + skim, + answers_in.as_deref(), + min_accuracy, + ) } Err(e) => { eprintln!("FAIL: {e}"); @@ -156,3 +180,127 @@ fn collect_yaml( } Ok(()) } + +/// The offline counterpart to `content::load_needs_from_gitea` - +/// `needs.yaml` sits at the repo root beside `aggregates.yaml`. +fn load_needs_from_dir(dir: &str) -> anyhow::Result> { + let needs_path = std::path::Path::new(dir) + .parent() + .unwrap_or_else(|| std::path::Path::new(".")) + .join("needs.yaml"); + if !needs_path.exists() { + return Ok(None); + } + std::fs::read_to_string(&needs_path) + .map(Some) + .map_err(|e| anyhow::anyhow!("reading {}: {e}", needs_path.display())) +} + +/// The business-needs pass (see `portal::needs`): structure always, +/// then the wording tasks written out and/or an engine's answers +/// scored, when asked. A repo without `needs.yaml` skips all of it. +fn check_needs( + raw: Option<&str>, + questions: &std::collections::HashMap, + aggregates_map: &std::collections::HashMap, + tasks_out: Option<&str>, + skim: bool, + answers_in: Option<&str>, + min_accuracy: f64, +) -> anyhow::Result<()> { + let Some(raw) = raw else { + if tasks_out.is_some() || answers_in.is_some() { + eprintln!("FAIL: --needs-tasks/--needs-score given, but the repo has no needs.yaml"); + std::process::exit(1); + } + return Ok(()); + }; + let file = match needs::parse(raw) { + Ok(file) => file, + Err(e) => { + eprintln!("FAIL: {e}"); + std::process::exit(1); + } + }; + let findings = needs::check(&file, questions, aggregates_map); + let mut failed = 0; + for finding in &findings { + let level = match finding.level { + needs::Level::Fail => { + failed += 1; + "FAIL" + } + needs::Level::Warn => "WARN", + }; + eprintln!("{level}: need {:?}: {}", finding.need, finding.message); + } + for need in &file.needs { + if let Some(path) = needs::path_for(questions, need) { + let steps: Vec = path + .iter() + .map(|s| format!("{} [{}]", s.page, s.alternative)) + .collect(); + println!("need {:?}: {} -> {:?}", need.id, steps.join(" -> "), need.lands_in); + } + } + if failed == 0 { + let planned = file.needs.iter().filter(|n| n.planned).count(); + println!( + "OK: {} need(s) structurally met, {} planned, {} persona(s)", + file.needs.len() - planned, + planned, + file.personas.len() + ); + } + // The wording pass still runs for the needs that do have a path - + // a closed flow elsewhere is no reason to stop grading the rest. + + let tasks = needs::tasks(&file, questions, skim); + if let Some(out) = tasks_out { + let mut lines = String::new(); + for task in &tasks { + lines.push_str(&serde_json::to_string(task)?); + lines.push('\n'); + } + std::fs::write(out, lines).map_err(|e| anyhow::anyhow!("writing {out}: {e}"))?; + println!("wrote {} wording task(s) to {out}", tasks.len()); + } + if let Some(path) = answers_in { + let raw = std::fs::read_to_string(path).map_err(|e| anyhow::anyhow!("reading {path}: {e}"))?; + let mut answers = Vec::new(); + for (n, line) in raw.lines().enumerate().filter(|(_, l)| !l.trim().is_empty()) { + answers.push( + serde_json::from_str::(line) + .map_err(|e| anyhow::anyhow!("{path}:{}: {e}", n + 1))?, + ); + } + let score = needs::score(&tasks, &answers, 0.6); + for (id, picked, expected, confidence) in &score.misroutes { + eprintln!( + "MISROUTE: {id}: picked {picked:?}{}, the need is met by {expected:?}", + confidence.map(|c| format!(" ({c:.2})")).unwrap_or_default() + ); + } + for (id, confidence) in &score.hesitant { + eprintln!("HESITANT: {id}: right, at {confidence:.2}"); + } + for id in &score.unanswered { + eprintln!("UNANSWERED: {id}"); + } + println!( + "wording: {}/{} routed right ({:.0}%)", + score.correct, + score.total, + score.accuracy() * 100.0 + ); + if score.accuracy() < min_accuracy { + eprintln!("FAIL: below --min-accuracy {min_accuracy}"); + std::process::exit(1); + } + } + if failed > 0 { + eprintln!("FAIL: {failed} unmet business need finding(s)"); + std::process::exit(1); + } + Ok(()) +} diff --git a/src/content.rs b/src/content.rs index a6fd96e..4fea701 100644 --- a/src/content.rs +++ b/src/content.rs @@ -719,6 +719,20 @@ pub fn validate_relay_url(relay: &str) -> anyhow::Result<()> { Ok(()) } +/// Fetches `needs.yaml` from the content repo root - the business's +/// needs spec `question_lint` checks the pages against (see +/// `crate::needs`). Optional: a fetch failure means the repo declares +/// none. The runtime never reads it. +#[cfg(feature = "ssr")] +pub async fn load_needs_from_gitea(repo_url: &str, branch: &str) -> anyhow::Result> { + let (owner, repo) = parse_owner_repo(repo_url)?; + let api_base = gitea_api_base(repo_url)?; + let client = openidconnect::reqwest::Client::new(); + Ok(fetch_gitea_file(&client, &api_base, &owner, &repo, branch, "needs.yaml") + .await + .ok()) +} + /// Fetches and parses `site.yaml` from the content repo root. A fetch /// failure (typically 404 - the file is optional) yields the default /// config; a file that exists but doesn't parse or validate is a real diff --git a/src/lib.rs b/src/lib.rs index 0d91949..38db8da 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,9 @@ pub mod aggregates; #[cfg(feature = "ssr")] pub mod announce; +#[cfg(feature = "ssr")] +pub mod needs; + #[cfg(feature = "ssr")] pub mod server; diff --git a/src/needs.rs b/src/needs.rs new file mode 100644 index 0000000..ad169ff --- /dev/null +++ b/src/needs.rs @@ -0,0 +1,762 @@ +//! A business's needs, stated apart from how its pages word them. +//! +//! A content repo may ship a `needs.yaml` at its root: each *need* says +//! who wants what, where they enter, which bucket the answer must land +//! in, and which group has to be able to carry it to a finished state. +//! Two things are checked against it: +//! +//! - **Structure** (`check`): deterministic. A path of submissions +//! leads from the entry page to the bucket, within an effort budget; +//! the handling group has a desk over that bucket; every finished +//! state is reachable through the moves that desk offers, and no +//! record strands on the way. +//! - **Wording** (`tasks` / `score`): each *persona* is a visitor in +//! their own words. Every page on their path becomes one typed +//! `choice` question for an external answering engine - the page's +//! question, its alternatives as the options - and the engine's picks +//! are scored against the alternatives that actually meet the need. +//! A misroute is a wording defect: the page said one thing and the +//! visitor heard another. +//! +//! The engine is deliberately outside this crate: tasks go out as +//! JSONL, answers come back as JSONL, so any classifier or LLM can sit +//! in between and be compared on the same tasks. +#![cfg(feature = "ssr")] + +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; + +use serde::{Deserialize, Serialize}; + +use crate::aggregates::AggregateSchema; +use crate::content::{action_target_id, Alternative, Question}; + +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NeedsFile { + #[serde(default)] + pub needs: Vec, + #[serde(default)] + pub personas: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Need { + pub id: String, + /// The kind of visitor - prose, for the reader of the report. + pub who: String, + pub wants: String, + /// The page they arrive on. + #[serde(default = "root")] + pub enters: String, + /// The group they hold when they arrive; unset means anonymous. + #[serde(default)] + pub enters_as: Option, + /// The alternative on the entry page that serves this need, when + /// several record into the same bucket. + #[serde(default)] + pub via: Option, + /// The bucket their answer must be recorded into. + pub lands_in: String, + /// The group that must be able to see the record and move it. + pub handled_by: String, + /// The states that count as handled. + pub done_when: Vec, + /// Submissions the visitor may be asked for before the record + /// exists. + #[serde(default = "default_max_steps")] + pub max_steps: usize, + /// Required fields across the whole path - the onboarding effort + /// budget. + #[serde(default)] + pub max_fields: Option, + /// Declared, and knowingly not open yet: what is missing reports + /// as a warning, so the gap stays visible without failing CI. + #[serde(default)] + pub planned: bool, +} + +fn root() -> String { + "/".to_string() +} + +fn default_max_steps() -> usize { + 2 +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Persona { + pub id: String, + pub need: String, + /// The visitor's situation in their own words - never the page's. + pub text: String, +} + +pub fn parse(raw: &str) -> anyhow::Result { + serde_yaml::from_str(raw).map_err(|e| anyhow::anyhow!("parsing needs.yaml: {e}")) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Level { + Fail, + Warn, +} + +#[derive(Debug)] +pub struct Finding { + pub need: String, + pub level: Level, + pub message: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Step { + pub page: String, + pub alternative: String, +} + +/// One alternative on a page that still meets the need, with the +/// cheapest way on from it. +struct Viable { + alternative: String, + path: Vec, + fields: usize, + records_with_action: Option, +} + +type Questions = HashMap; + +fn visible(q: &Question, as_group: Option<&str>) -> bool { + match &q.qualifies { + None => true, + Some(group) => as_group == Some(group.as_str()), + } +} + +fn required_fields(alt: &Alternative) -> usize { + alt.features + .iter() + .flat_map(|f| &f.requirements) + .filter(|r| !r.optional) + .count() +} + +/// The alternatives on `page` from which the need is still met within +/// `remaining` submissions. +fn viable( + questions: &Questions, + need: &Need, + page: &Question, + remaining: usize, + entry: bool, + include_disabled: bool, + trail: &mut Vec, +) -> Vec { + let mut out = Vec::new(); + if remaining == 0 { + return out; + } + for alt in page.alternatives.iter().filter(|a| include_disabled || !a.disabled) { + if entry && need.via.as_deref().is_some_and(|via| via != alt.name) { + continue; + } + let step = Step { + page: page.id.clone(), + alternative: alt.name.clone(), + }; + let fields = required_fields(alt); + if alt.record_as.as_deref() == Some(need.lands_in.as_str()) { + out.push(Viable { + alternative: alt.name.clone(), + path: vec![step], + fields, + records_with_action: Some(alt.action.is_some()), + }); + continue; + } + let Some(target) = alt + .action + .as_deref() + .and_then(|a| action_target_id(questions, a)) + .and_then(|id| questions.get(&id)) + else { + continue; + }; + if trail.contains(&target.id) || !visible(target, need.enters_as.as_deref()) { + continue; + } + trail.push(target.id.clone()); + let onward = viable(questions, need, target, remaining - 1, false, include_disabled, trail); + trail.pop(); + if let Some(best) = onward.into_iter().min_by_key(|v| (v.path.len(), v.fields)) { + let mut path = vec![step]; + path.extend(best.path); + out.push(Viable { + alternative: alt.name.clone(), + path, + fields: fields + best.fields, + records_with_action: best.records_with_action, + }); + } + } + out +} + +fn entry_viable(questions: &Questions, need: &Need, include_disabled: bool) -> Option> { + let page = questions.get(&need.enters)?; + let mut trail = vec![page.id.clone()]; + Some(viable(questions, need, page, need.max_steps, true, include_disabled, &mut trail)) +} + +/// The cheapest path that meets the need, if any. +pub fn path_for(questions: &Questions, need: &Need) -> Option> { + entry_viable(questions, need, false)? + .into_iter() + .min_by_key(|v| (v.path.len(), v.fields)) + .map(|v| v.path) +} + +struct Desk { + page: String, + group: Option, + moves: Vec<(String, String)>, +} + +fn desks_over(questions: &Questions, bucket: &str) -> Vec { + let mut desks = Vec::new(); + for q in questions.values() { + for resource in q + .alternatives + .iter() + .flat_map(|a| &a.features) + .filter_map(|f| f.resource.as_ref()) + .filter(|r| r.bucket() == Some(bucket)) + { + desks.push(Desk { + page: q.id.clone(), + group: resource.requires_group.clone().or_else(|| q.qualifies.clone()), + moves: resource + .transitions + .iter() + .map(|t| (t.from.clone(), t.to.clone())) + .collect(), + }); + } + } + desks.sort_by(|a, b| a.page.cmp(&b.page)); + desks +} + +/// Self-service moves into `bucket` - a visitor holding their own +/// record's link, no group involved (unsubscribe). +fn self_moves(questions: &Questions, bucket: &str) -> Vec { + questions + .values() + .flat_map(|q| &q.alternatives) + .filter_map(|a| a.self_transition.as_ref()) + .filter(|s| s.bucket == bucket) + .map(|s| s.to.clone()) + .collect() +} + +/// Structural check of every need. `Fail` findings mean the business +/// need is not met by the content as written. +pub fn check( + file: &NeedsFile, + questions: &Questions, + aggregates: &HashMap, +) -> Vec { + let mut findings = Vec::new(); + let mut seen = HashSet::new(); + for need in &file.needs { + let mut report = |level: Level, message: String| { + findings.push(Finding { + need: need.id.clone(), + level: if need.planned { Level::Warn } else { level }, + message: if need.planned && level == Level::Fail { + format!("planned, not met yet: {message}") + } else { + message + }, + }) + }; + if !seen.insert(need.id.as_str()) { + report(Level::Fail, "declared more than once".to_string()); + continue; + } + + // The visitor's side: can they get their answer recorded? + match questions.get(&need.enters) { + None => report(Level::Fail, format!("entry page {:?} does not exist", need.enters)), + Some(page) if !visible(page, need.enters_as.as_deref()) => report( + Level::Fail, + format!( + "entry page {:?} is gated on {:?}, which this visitor does not hold", + need.enters, page.qualifies + ), + ), + Some(page) => { + if page.responsible.is_none() { + report( + Level::Warn, + format!("entry page {:?} names no one responsible to turn to", need.enters), + ); + } + let best = entry_viable(questions, need, false) + .unwrap_or_default() + .into_iter() + .min_by_key(|v| (v.path.len(), v.fields)); + match best { + None if entry_viable(questions, need, true).is_some_and(|v| !v.is_empty()) => { + let blocked = entry_viable(questions, need, true) + .unwrap_or_default() + .into_iter() + .min_by_key(|v| (v.path.len(), v.fields)) + .map(|v| v.path) + .unwrap_or_default(); + let at = blocked + .iter() + .find(|s| { + questions[&s.page] + .alternatives + .iter() + .any(|a| a.name == s.alternative && a.disabled) + }) + .map(|s| format!("{:?} on {}", s.alternative, s.page)) + .unwrap_or_default(); + report( + Level::Fail, + format!( + "the path exists but is closed: {at} is `disabled`, so the answer is advertised and refused" + ), + ) + } + None => report( + Level::Fail, + format!( + "no path from {:?}{} records into {:?} within {} submission(s)", + need.enters, + need.via + .as_ref() + .map(|v| format!(" via {v:?}")) + .unwrap_or_default(), + need.lands_in, + need.max_steps + ), + ), + Some(best) => { + if let Some(budget) = need.max_fields { + if best.fields > budget { + report( + Level::Fail, + format!( + "the path asks for {} required field(s), over the budget of {budget}", + best.fields + ), + ); + } + } + if best.records_with_action == Some(false) { + report( + Level::Warn, + "the recording alternative has no `action` - the visitor is never told what happens next".to_string(), + ); + } + } + } + } + } + + // The business's side: can the handling group finish it? + let Some(schema) = aggregates.get(&need.lands_in) else { + report( + Level::Fail, + format!("aggregates.yaml declares no state machine for {:?}", need.lands_in), + ); + continue; + }; + let desks = desks_over(questions, &need.lands_in); + let own: Vec<&Desk> = desks + .iter() + .filter(|d| d.group.as_deref() == Some(need.handled_by.as_str())) + .collect(); + if own.is_empty() { + let others: Vec = desks + .iter() + .map(|d| format!("{} ({})", d.page, d.group.as_deref().unwrap_or("ungated"))) + .collect(); + report( + Level::Fail, + format!( + "no desk gated on {:?} reads {:?}{}", + need.handled_by, + need.lands_in, + if others.is_empty() { + String::new() + } else { + format!(" - it is read by {}", others.join(", ")) + } + ), + ); + continue; + } + let selfs = self_moves(questions, &need.lands_in); + let exits = |state: &str| -> Vec { + let allowed = schema.allowed(state); + let mut to: Vec = own + .iter() + .flat_map(|d| &d.moves) + .filter(|(from, to)| from == state && allowed.contains(to)) + .map(|(_, to)| to.clone()) + .collect(); + to.extend(selfs.iter().filter(|to| allowed.contains(to)).cloned()); + to + }; + let done: HashSet<&str> = need.done_when.iter().map(String::as_str).collect(); + // Reachability walks the whole machine (a finished state may + // lead on to another: subscribed, then unsubscribed). Stranding + // only counts on the way to the first finished state - what + // happens to a handled record afterwards is not this need. + let mut reached: HashSet = HashSet::from([schema.initial.clone()]); + let mut queue = VecDeque::from([(schema.initial.clone(), false)]); + let mut stranded = Vec::new(); + while let Some((state, past_done)) = queue.pop_front() { + let past_done = past_done || done.contains(state.as_str()); + let next = exits(&state); + if next.is_empty() && !past_done { + stranded.push(state.clone()); + } + for to in next { + if reached.insert(to.clone()) { + queue.push_back((to, past_done)); + } + } + } + for state in &need.done_when { + if !schema.has_state(state) { + report( + Level::Fail, + format!("done_when names {state:?}, which {:?} has no such state", need.lands_in), + ); + } else if !reached.contains(state) { + report( + Level::Fail, + format!( + "{:?} cannot move a record to {state:?} - no desk of theirs offers the moves", + need.handled_by + ), + ); + } + } + stranded.sort(); + for state in stranded { + report( + Level::Fail, + format!( + "records strand in {state:?}: {:?} is offered no move out of it", + need.handled_by + ), + ); + } + } + + let need_ids: HashSet<&str> = file.needs.iter().map(|n| n.id.as_str()).collect(); + for persona in &file.personas { + if !need_ids.contains(persona.need.as_str()) { + findings.push(Finding { + need: persona.need.clone(), + level: Level::Fail, + message: format!("persona {:?} points at a need that is not declared", persona.id), + }); + } + } + findings +} + +/// One decision for the answering engine - the shape is laya's +/// `predict(state, questions)` input, so an adapter is a loop and +/// nothing more; any other engine reads the same fields. +#[derive(Debug, Serialize)] +pub struct Task { + pub id: String, + pub persona: String, + pub need: String, + pub page: String, + pub state: BTreeMap<&'static str, String>, + pub questions: BTreeMap<&'static str, ChoiceQuestion>, + /// Every alternative that still meets the need from here. The + /// engine never sees this. + pub expect: Vec, +} + +#[derive(Debug, Serialize)] +pub struct ChoiceQuestion { + #[serde(rename = "type")] + pub kind: &'static str, + pub instructions: String, + pub criteria: BTreeMap, +} + +fn plain(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") +} + +/// Every (persona, page-on-their-path) pair with a real choice on it. +/// `skim` shows the engine only each alternative's heading - what a +/// visitor who never reads the description has to go on. +pub fn tasks(file: &NeedsFile, questions: &Questions, skim: bool) -> Vec { + let mut out = Vec::new(); + for persona in &file.personas { + let Some(need) = file.needs.iter().find(|n| n.id == persona.need) else { + continue; + }; + let Some(path) = path_for(questions, need) else { + continue; + }; + for (i, step) in path.iter().enumerate() { + let page = &questions[&step.page]; + let open: Vec<&Alternative> = page.alternatives.iter().filter(|a| !a.disabled).collect(); + if open.len() < 2 { + continue; + } + let mut trail: Vec = path[..=i].iter().map(|s| s.page.clone()).collect(); + let expect: Vec = viable(questions, need, page, need.max_steps - i, i == 0, false, &mut trail) + .into_iter() + .map(|v| v.alternative) + .collect(); + let criteria = open + .iter() + .map(|a| { + let description = plain(&a.description); + ( + a.name.clone(), + if skim || description.is_empty() { a.name.clone() } else { description }, + ) + }) + .collect(); + out.push(Task { + id: format!("{}@{}", persona.id, step.page), + persona: persona.id.clone(), + need: need.id.clone(), + page: step.page.clone(), + state: BTreeMap::from([("body", plain(&persona.text))]), + questions: BTreeMap::from([( + "pick", + ChoiceQuestion { + kind: "choice", + instructions: format!( + "A visitor reads the page question {:?}. Which alternative do they choose?", + page.name + ), + criteria, + }, + )]), + expect, + }); + } + } + out +} + +#[derive(Debug, Deserialize)] +pub struct Answer { + pub id: String, + pub choice: String, + #[serde(default)] + pub confidence: Option, +} + +#[derive(Debug, Default)] +pub struct Score { + pub total: usize, + pub correct: usize, + /// (task id, picked, expected, confidence) + pub misroutes: Vec<(String, String, Vec, Option)>, + /// Right, but the engine was under `hesitant_below` - wording that + /// works and is still close to a coin flip. + pub hesitant: Vec<(String, f64)>, + pub unanswered: Vec, +} + +impl Score { + pub fn accuracy(&self) -> f64 { + if self.total == 0 { + 1.0 + } else { + self.correct as f64 / self.total as f64 + } + } +} + +pub fn score(tasks: &[Task], answers: &[Answer], hesitant_below: f64) -> Score { + let by_id: HashMap<&str, &Answer> = answers.iter().map(|a| (a.id.as_str(), a)).collect(); + let mut score = Score { + total: tasks.len(), + ..Score::default() + }; + for task in tasks { + match by_id.get(task.id.as_str()) { + None => score.unanswered.push(task.id.clone()), + Some(answer) if task.expect.contains(&answer.choice) => { + score.correct += 1; + if let Some(c) = answer.confidence.filter(|c| *c < hesitant_below) { + score.hesitant.push((task.id.clone(), c)); + } + } + Some(answer) => score.misroutes.push(( + task.id.clone(), + answer.choice.clone(), + task.expect.clone(), + answer.confidence, + )), + } + } + score +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::aggregates::parse_aggregates_yaml; + use crate::content::build_questions; + + fn site(desk_moves: &str) -> (Questions, HashMap) { + let files = vec![ + ( + "index.yaml".to_string(), + "name: What are you here for?\nresponsible: { name: A, contact: a@b.c }\nalternatives:\n - name: Hire us\n description: Bring a problem.\n action: /thanks\n record_as: projects\n features:\n - name: Brief\n requirements:\n - name: email\n type: email\n - name: brief\n type: textarea\n - name: budget\n optional: true\n - name: Follow along\n description: Notes by mail.\n action: /thanks\n record_as: subscribers\n".to_string(), + ), + ("thanks.yaml".to_string(), "name: Thanks\nfollowup: true\n".to_string()), + ( + "desk/index.yaml".to_string(), + format!("name: Desk\nqualifies: owners\nalternatives:\n - name: Projects\n features:\n - name: \"\"\n resource:\n source: {{ kind: kv, bucket: projects }}\n requires_group: owners\n transitions:\n{desk_moves}"), + ), + ]; + let aggregates = parse_aggregates_yaml( + "aggregates:\n - bucket: projects\n initial: open\n states:\n open: { event: submitted }\n quoted: { event: quoted }\n won: { event: won }\n lost: { event: lost }\n transitions:\n open: [quoted, lost]\n quoted: [won, lost]\n - bucket: subscribers\n attended_by: a mailer\n initial: open\n states:\n open: { event: subscribed }\n transitions: {}\n", + ) + .unwrap(); + (build_questions(&files).unwrap(), aggregates) + } + + const FULL_DESK: &str = " - { to: quoted, label: Quote }\n - { to: lost, label: Lose }\n - { from: quoted, to: won, label: Win }\n - { from: quoted, to: lost, label: Lose }\n"; + + fn needs(extra: &str) -> NeedsFile { + parse(&format!( + "needs:\n - id: hire\n who: a client\n wants: work done\n lands_in: projects\n handled_by: owners\n done_when: [won, lost]\n{extra}personas:\n - id: founder\n need: hire\n text: I need someone to build our booking tool.\n" + )) + .unwrap() + } + + fn fails(findings: &[Finding]) -> Vec<&str> { + findings + .iter() + .filter(|f| f.level == Level::Fail) + .map(|f| f.message.as_str()) + .collect() + } + + #[test] + fn a_met_need_is_silent() { + let (questions, aggregates) = site(FULL_DESK); + let findings = check(&needs(" max_fields: 2\n"), &questions, &aggregates); + assert!(findings.is_empty(), "{findings:?}"); + } + + #[test] + fn effort_budget_counts_required_fields_only() { + let (questions, aggregates) = site(FULL_DESK); + let findings = check(&needs(" max_fields: 1\n"), &questions, &aggregates); + assert_eq!(fails(&findings), vec!["the path asks for 2 required field(s), over the budget of 1"]); + } + + #[test] + fn a_desk_missing_a_move_strands_records_and_cannot_finish() { + // The desk can quote, but never close a quoted job. + let (questions, aggregates) = site(" - { to: quoted, label: Quote }\n - { to: lost, label: Lose }\n"); + let findings = check(&needs(""), &questions, &aggregates); + assert_eq!( + fails(&findings), + vec![ + "\"owners\" cannot move a record to \"won\" - no desk of theirs offers the moves", + "records strand in \"quoted\": \"owners\" is offered no move out of it", + ] + ); + } + + #[test] + fn a_finished_state_may_lead_on_to_another() { + // quoted counts as handled here, yet won must still be + // reachable past it; lost-after-quoted is not this need's + // business, so nothing strands. + let (questions, aggregates) = site(FULL_DESK); + let mut file = needs(""); + file.needs[0].done_when = vec!["quoted".to_string(), "won".to_string(), "lost".to_string()]; + let findings = check(&file, &questions, &aggregates); + assert!(findings.is_empty(), "{findings:?}"); + } + + #[test] + fn a_disabled_alternative_is_named_as_the_reason() { + let (mut questions, aggregates) = site(FULL_DESK); + questions.get_mut("/").unwrap().alternatives[0].disabled = true; + let findings = check(&needs(""), &questions, &aggregates); + assert_eq!( + fails(&findings), + vec!["the path exists but is closed: \"Hire us\" on / is `disabled`, so the answer is advertised and refused"] + ); + } + + #[test] + fn the_wrong_group_holding_the_desk_is_a_failure() { + let (questions, aggregates) = site(FULL_DESK); + let mut file = needs(""); + file.needs[0].handled_by = "sales".to_string(); + let findings = check(&file, &questions, &aggregates); + assert_eq!( + fails(&findings), + vec!["no desk gated on \"sales\" reads \"projects\" - it is read by /desk (owners)"] + ); + } + + #[test] + fn via_pins_the_entry_alternative() { + let (questions, aggregates) = site(FULL_DESK); + let mut file = needs(""); + file.needs[0].via = Some("Follow along".to_string()); + let findings = check(&file, &questions, &aggregates); + assert_eq!( + fails(&findings), + vec!["no path from \"/\" via \"Follow along\" records into \"projects\" within 2 submission(s)"] + ); + } + + #[test] + fn tasks_carry_the_page_as_a_choice_and_score_separates_misroutes() { + let (questions, _) = site(FULL_DESK); + let file = needs(""); + let tasks = tasks(&file, &questions, false); + let skimmed = super::tasks(&file, &questions, true); + assert_eq!(skimmed[0].questions["pick"].criteria["Follow along"], "Follow along"); + assert_eq!(tasks.len(), 1); + let task = &tasks[0]; + assert_eq!(task.id, "founder@/"); + assert_eq!(task.expect, vec!["Hire us"]); + let q = &task.questions["pick"]; + assert_eq!(q.kind, "choice"); + assert_eq!(q.criteria["Follow along"], "Notes by mail."); + assert!(q.instructions.contains("What are you here for?")); + + let right = vec![Answer { id: "founder@/".into(), choice: "Hire us".into(), confidence: Some(0.55) }]; + let s = score(&tasks, &right, 0.6); + assert_eq!((s.correct, s.misroutes.len(), s.hesitant.len()), (1, 0, 1)); + + let wrong = vec![Answer { id: "founder@/".into(), choice: "Follow along".into(), confidence: Some(0.9) }]; + let s = score(&tasks, &wrong, 0.6); + assert_eq!((s.correct, s.misroutes.len()), (0, 1)); + assert_eq!(s.accuracy(), 0.0); + + assert_eq!(score(&tasks, &[], 0.6).unanswered, vec!["founder@/"]); + } +}