iris: the lint over a portal site, as its own repo
question_lint becomes `iris check`, needs_replay becomes `iris replay`, and the needs module and local-checkout loaders come with them. Portal is a library dependency pinned to the release iris matches (v0.3.36), so every type iris reads is the site's own and the two never disagree about what a page is. `iris --path questions` still works, so a content repo's one-line CI needs only a new path. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01L4jrCgLiKKHAEFuZUJjckH
This commit is contained in:
Bendik Aagaard Lynghaug
co-authored by
Claude Fable 5.1
Claude Fable 5.1
commit
73906cdf35
+968
@@ -0,0 +1,968 @@
|
||||
//! A business's needs, stated apart from how its pages word them.
|
||||
//!
|
||||
//! A content repo may ship a `needs.yaml` at its root: each *need* says
|
||||
//! who wants what, where they enter, which bucket the answer must land
|
||||
//! in, and which group has to be able to carry it to a finished state.
|
||||
//! Two things are checked against it:
|
||||
//!
|
||||
//! - **Structure** (`check`): deterministic. A path of submissions
|
||||
//! leads from the entry page to the bucket, within an effort budget;
|
||||
//! the handling group has a desk over that bucket; every finished
|
||||
//! state is reachable through the moves that desk offers, and no
|
||||
//! record strands on the way.
|
||||
//! - **Wording** (`tasks` / `score`): each *persona* is a visitor in
|
||||
//! their own words. Every page on their path becomes one typed
|
||||
//! `choice` question for an external answering engine - the page's
|
||||
//! question, its alternatives as the options - and the engine's picks
|
||||
//! are scored against the alternatives that actually meet the need.
|
||||
//! A misroute is a wording defect: the page said one thing and the
|
||||
//! visitor heard another.
|
||||
//!
|
||||
//! The engine is deliberately outside this crate: tasks go out as
|
||||
//! JSONL, answers come back as JSONL, so any classifier or LLM can sit
|
||||
//! in between and be compared on the same tasks.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use portal::aggregates::AggregateSchema;
|
||||
use portal::content::{action_target_id, Alternative, Question};
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct NeedsFile {
|
||||
#[serde(default)]
|
||||
pub needs: Vec<Need>,
|
||||
#[serde(default)]
|
||||
pub personas: Vec<Persona>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Need {
|
||||
pub id: String,
|
||||
/// The kind of visitor - prose, for the reader of the report.
|
||||
pub who: String,
|
||||
pub wants: String,
|
||||
/// The page they arrive on.
|
||||
#[serde(default = "root")]
|
||||
pub enters: String,
|
||||
/// The group they hold when they arrive; unset means anonymous.
|
||||
#[serde(default)]
|
||||
pub enters_as: Option<String>,
|
||||
/// The alternative on the entry page that serves this need, when
|
||||
/// several record into the same bucket.
|
||||
#[serde(default)]
|
||||
pub via: Option<String>,
|
||||
/// The bucket their answer must be recorded into.
|
||||
pub lands_in: String,
|
||||
/// The group that must be able to see the record and move it.
|
||||
pub handled_by: String,
|
||||
/// The states that count as handled.
|
||||
pub done_when: Vec<String>,
|
||||
/// Submissions the visitor may be asked for before the record
|
||||
/// exists.
|
||||
#[serde(default = "default_max_steps")]
|
||||
pub max_steps: usize,
|
||||
/// Required fields across the whole path - the onboarding effort
|
||||
/// budget.
|
||||
#[serde(default)]
|
||||
pub max_fields: Option<usize>,
|
||||
/// Declared, and knowingly not open yet: what is missing reports
|
||||
/// as a warning, so the gap stays visible without failing CI.
|
||||
#[serde(default)]
|
||||
pub planned: bool,
|
||||
/// The finished states that are a good outcome for the business -
|
||||
/// a subset of `done_when`. "lost" is finished; "won" is success.
|
||||
/// What a simulation or a live bucket report counts toward.
|
||||
#[serde(default)]
|
||||
pub success: Vec<String>,
|
||||
/// Every stage the business said a case goes through, in its own
|
||||
/// order, first to last. Written from how the business describes
|
||||
/// its work, before any state machine exists, so that a machine
|
||||
/// which quietly skips a stage ("we go and look, then we quote"
|
||||
/// becoming open -> quoted) fails instead of passing.
|
||||
#[serde(default)]
|
||||
pub stages: Vec<String>,
|
||||
/// Other groups whose desks carry some of this need's moves: a
|
||||
/// site manager closing what a project manager opened. Their
|
||||
/// buttons count toward reaching `done_when` exactly as
|
||||
/// `handled_by`'s do. `handled_by` must still hold a desk of its
|
||||
/// own - someone has to be answerable for the record.
|
||||
#[serde(default)]
|
||||
pub also_moved_by: Vec<String>,
|
||||
}
|
||||
|
||||
fn root() -> String {
|
||||
"/".to_string()
|
||||
}
|
||||
|
||||
fn default_max_steps() -> usize {
|
||||
2
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Persona {
|
||||
pub id: String,
|
||||
pub need: String,
|
||||
/// The visitor's situation in their own words - never the page's.
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
pub fn parse(raw: &str) -> anyhow::Result<NeedsFile> {
|
||||
serde_yaml::from_str(raw).map_err(|e| anyhow::anyhow!("parsing needs.yaml: {e}"))
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Level {
|
||||
Fail,
|
||||
Warn,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Finding {
|
||||
pub need: String,
|
||||
pub level: Level,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Step {
|
||||
pub page: String,
|
||||
pub alternative: String,
|
||||
}
|
||||
|
||||
/// One alternative on a page that still meets the need, with the
|
||||
/// cheapest way on from it.
|
||||
struct Viable {
|
||||
alternative: String,
|
||||
path: Vec<Step>,
|
||||
fields: usize,
|
||||
records_with_action: Option<bool>,
|
||||
}
|
||||
|
||||
type Questions = HashMap<String, Question>;
|
||||
|
||||
fn visible(q: &Question, as_group: Option<&str>) -> bool {
|
||||
match &q.qualifies {
|
||||
None => true,
|
||||
Some(group) => as_group == Some(group.as_str()),
|
||||
}
|
||||
}
|
||||
|
||||
fn required_fields(alt: &Alternative) -> usize {
|
||||
alt.features
|
||||
.iter()
|
||||
.flat_map(|f| &f.requirements)
|
||||
.filter(|r| !r.optional)
|
||||
.count()
|
||||
}
|
||||
|
||||
/// The alternatives on `page` from which the need is still met within
|
||||
/// `remaining` submissions.
|
||||
fn viable(
|
||||
questions: &Questions,
|
||||
need: &Need,
|
||||
page: &Question,
|
||||
remaining: usize,
|
||||
entry: bool,
|
||||
include_disabled: bool,
|
||||
trail: &mut Vec<String>,
|
||||
) -> Vec<Viable> {
|
||||
let mut out = Vec::new();
|
||||
if remaining == 0 {
|
||||
return out;
|
||||
}
|
||||
for alt in page.alternatives.iter().filter(|a| include_disabled || !a.disabled) {
|
||||
if entry && need.via.as_deref().is_some_and(|via| via != alt.name) {
|
||||
continue;
|
||||
}
|
||||
let step = Step {
|
||||
page: page.id.clone(),
|
||||
alternative: alt.name.clone(),
|
||||
};
|
||||
let fields = required_fields(alt);
|
||||
if alt.record_as.as_deref() == Some(need.lands_in.as_str()) {
|
||||
out.push(Viable {
|
||||
alternative: alt.name.clone(),
|
||||
path: vec![step],
|
||||
fields,
|
||||
records_with_action: Some(alt.action.is_some()),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
let Some(target) = alt
|
||||
.action
|
||||
.as_deref()
|
||||
.and_then(|a| action_target_id(questions, a))
|
||||
.and_then(|id| questions.get(&id))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if trail.contains(&target.id) || !visible(target, need.enters_as.as_deref()) {
|
||||
continue;
|
||||
}
|
||||
trail.push(target.id.clone());
|
||||
let onward = viable(questions, need, target, remaining - 1, false, include_disabled, trail);
|
||||
trail.pop();
|
||||
if let Some(best) = onward.into_iter().min_by_key(|v| (v.path.len(), v.fields)) {
|
||||
let mut path = vec![step];
|
||||
path.extend(best.path);
|
||||
out.push(Viable {
|
||||
alternative: alt.name.clone(),
|
||||
path,
|
||||
fields: fields + best.fields,
|
||||
records_with_action: best.records_with_action,
|
||||
});
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn entry_viable(questions: &Questions, need: &Need, include_disabled: bool) -> Option<Vec<Viable>> {
|
||||
let page = questions.get(&need.enters)?;
|
||||
let mut trail = vec![page.id.clone()];
|
||||
Some(viable(questions, need, page, need.max_steps, true, include_disabled, &mut trail))
|
||||
}
|
||||
|
||||
/// The cheapest path that meets the need, if any.
|
||||
pub fn path_for(questions: &Questions, need: &Need) -> Option<Vec<Step>> {
|
||||
entry_viable(questions, need, false)?
|
||||
.into_iter()
|
||||
.min_by_key(|v| (v.path.len(), v.fields))
|
||||
.map(|v| v.path)
|
||||
}
|
||||
|
||||
struct Desk {
|
||||
page: String,
|
||||
group: Option<String>,
|
||||
/// (from, to, button label)
|
||||
moves: Vec<(String, String, String)>,
|
||||
}
|
||||
|
||||
fn desks_over(questions: &Questions, bucket: &str) -> Vec<Desk> {
|
||||
let mut desks = Vec::new();
|
||||
for q in questions.values() {
|
||||
for resource in q
|
||||
.alternatives
|
||||
.iter()
|
||||
.flat_map(|a| &a.features)
|
||||
.filter_map(|f| f.resource.as_ref())
|
||||
.filter(|r| r.bucket() == Some(bucket))
|
||||
{
|
||||
desks.push(Desk {
|
||||
page: q.id.clone(),
|
||||
group: resource.requires_group.clone().or_else(|| q.qualifies.clone()),
|
||||
moves: resource
|
||||
.transitions
|
||||
.iter()
|
||||
.map(|t| (t.from.clone(), t.to.clone(), t.label.clone()))
|
||||
.collect(),
|
||||
});
|
||||
}
|
||||
}
|
||||
desks.sort_by(|a, b| a.page.cmp(&b.page));
|
||||
desks
|
||||
}
|
||||
|
||||
/// Self-service moves into `bucket` - a visitor holding their own
|
||||
/// record's link, no group involved (unsubscribe).
|
||||
fn self_moves(questions: &Questions, bucket: &str) -> Vec<String> {
|
||||
questions
|
||||
.values()
|
||||
.flat_map(|q| &q.alternatives)
|
||||
.filter_map(|a| a.self_transition.as_ref())
|
||||
.filter(|s| s.bucket == bucket)
|
||||
.map(|s| s.to.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Structural check of every need. `Fail` findings mean the business
|
||||
/// need is not met by the content as written.
|
||||
pub fn check(
|
||||
file: &NeedsFile,
|
||||
questions: &Questions,
|
||||
aggregates: &HashMap<String, AggregateSchema>,
|
||||
) -> Vec<Finding> {
|
||||
let mut findings = Vec::new();
|
||||
let mut seen = HashSet::new();
|
||||
for need in &file.needs {
|
||||
let mut report = |level: Level, message: String| {
|
||||
findings.push(Finding {
|
||||
need: need.id.clone(),
|
||||
level: if need.planned { Level::Warn } else { level },
|
||||
message: if need.planned && level == Level::Fail {
|
||||
format!("planned, not met yet: {message}")
|
||||
} else {
|
||||
message
|
||||
},
|
||||
})
|
||||
};
|
||||
if !seen.insert(need.id.as_str()) {
|
||||
report(Level::Fail, "declared more than once".to_string());
|
||||
continue;
|
||||
}
|
||||
|
||||
// The visitor's side: can they get their answer recorded?
|
||||
match questions.get(&need.enters) {
|
||||
None => report(Level::Fail, format!("entry page {:?} does not exist", need.enters)),
|
||||
Some(page) if !visible(page, need.enters_as.as_deref()) => report(
|
||||
Level::Fail,
|
||||
format!(
|
||||
"entry page {:?} is gated on {:?}, which this visitor does not hold",
|
||||
need.enters, page.qualifies
|
||||
),
|
||||
),
|
||||
Some(page) => {
|
||||
if page.responsible.is_none() {
|
||||
report(
|
||||
Level::Warn,
|
||||
format!("entry page {:?} names no one responsible to turn to", need.enters),
|
||||
);
|
||||
}
|
||||
let best = entry_viable(questions, need, false)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.min_by_key(|v| (v.path.len(), v.fields));
|
||||
match best {
|
||||
None if entry_viable(questions, need, true).is_some_and(|v| !v.is_empty()) => {
|
||||
let blocked = entry_viable(questions, need, true)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.min_by_key(|v| (v.path.len(), v.fields))
|
||||
.map(|v| v.path)
|
||||
.unwrap_or_default();
|
||||
let at = blocked
|
||||
.iter()
|
||||
.find(|s| {
|
||||
questions[&s.page]
|
||||
.alternatives
|
||||
.iter()
|
||||
.any(|a| a.name == s.alternative && a.disabled)
|
||||
})
|
||||
.map(|s| format!("{:?} on {}", s.alternative, s.page))
|
||||
.unwrap_or_default();
|
||||
report(
|
||||
Level::Fail,
|
||||
format!(
|
||||
"the path exists but is closed: {at} is `disabled`, so the answer is advertised and refused"
|
||||
),
|
||||
)
|
||||
}
|
||||
None => report(
|
||||
Level::Fail,
|
||||
format!(
|
||||
"no path from {:?}{} records into {:?} within {} submission(s)",
|
||||
need.enters,
|
||||
need.via
|
||||
.as_ref()
|
||||
.map(|v| format!(" via {v:?}"))
|
||||
.unwrap_or_default(),
|
||||
need.lands_in,
|
||||
need.max_steps
|
||||
),
|
||||
),
|
||||
Some(best) => {
|
||||
if let Some(budget) = need.max_fields {
|
||||
if best.fields > budget {
|
||||
report(
|
||||
Level::Fail,
|
||||
format!(
|
||||
"the path asks for {} required field(s), over the budget of {budget}",
|
||||
best.fields
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
if best.records_with_action == Some(false) {
|
||||
report(
|
||||
Level::Warn,
|
||||
"the recording alternative has no `action` - the visitor is never told what happens next".to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The business's side: can the handling group finish it?
|
||||
let Some(schema) = aggregates.get(&need.lands_in) else {
|
||||
report(
|
||||
Level::Fail,
|
||||
format!("aggregates.yaml declares no state machine for {:?}", need.lands_in),
|
||||
);
|
||||
continue;
|
||||
};
|
||||
let desks = desks_over(questions, &need.lands_in);
|
||||
let own: Vec<&Desk> = desks
|
||||
.iter()
|
||||
.filter(|d| {
|
||||
d.group.as_deref().is_some_and(|g| g == need.handled_by || need.also_moved_by.iter().any(|m| m == g))
|
||||
})
|
||||
.collect();
|
||||
if !desks.iter().any(|d| d.group.as_deref() == Some(need.handled_by.as_str())) {
|
||||
let others: Vec<String> = desks
|
||||
.iter()
|
||||
.map(|d| format!("{} ({})", d.page, d.group.as_deref().unwrap_or("ungated")))
|
||||
.collect();
|
||||
report(
|
||||
Level::Fail,
|
||||
format!(
|
||||
"no desk gated on {:?} reads {:?}{}",
|
||||
need.handled_by,
|
||||
need.lands_in,
|
||||
if others.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" - it is read by {}", others.join(", "))
|
||||
}
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let selfs = self_moves(questions, &need.lands_in);
|
||||
let exits = |state: &str| -> Vec<String> {
|
||||
let allowed = schema.allowed(state);
|
||||
let mut to: Vec<String> = own
|
||||
.iter()
|
||||
.flat_map(|d| &d.moves)
|
||||
.filter(|(from, to, _)| from == state && allowed.contains(to))
|
||||
.map(|(_, to, _)| to.clone())
|
||||
.collect();
|
||||
to.extend(selfs.iter().filter(|to| allowed.contains(to)).cloned());
|
||||
to
|
||||
};
|
||||
let done: HashSet<&str> = need.done_when.iter().map(String::as_str).collect();
|
||||
// Reachability walks the whole machine (a finished state may
|
||||
// lead on to another: subscribed, then unsubscribed). Stranding
|
||||
// only counts on the way to the first finished state - what
|
||||
// happens to a handled record afterwards is not this need.
|
||||
let mut reached: HashSet<String> = HashSet::from([schema.initial.clone()]);
|
||||
let mut queue = VecDeque::from([(schema.initial.clone(), false)]);
|
||||
let mut stranded = Vec::new();
|
||||
while let Some((state, past_done)) = queue.pop_front() {
|
||||
let past_done = past_done || done.contains(state.as_str());
|
||||
let next = exits(&state);
|
||||
if next.is_empty() && !past_done {
|
||||
stranded.push(state.clone());
|
||||
}
|
||||
for to in next {
|
||||
if reached.insert(to.clone()) {
|
||||
queue.push_back((to, past_done));
|
||||
}
|
||||
}
|
||||
}
|
||||
for state in &need.done_when {
|
||||
if !schema.has_state(state) {
|
||||
report(
|
||||
Level::Fail,
|
||||
format!("done_when names {state:?}, which {:?} has no such state", need.lands_in),
|
||||
);
|
||||
} else if !reached.contains(state) {
|
||||
report(
|
||||
Level::Fail,
|
||||
format!(
|
||||
"{:?} cannot move a record to {state:?} - no desk of theirs offers the moves",
|
||||
need.handled_by
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
for stage in &need.stages {
|
||||
if !schema.has_state(stage) {
|
||||
report(
|
||||
Level::Fail,
|
||||
format!(
|
||||
"the business describes a stage {stage:?} that {:?} has no state for",
|
||||
need.lands_in
|
||||
),
|
||||
);
|
||||
} else if !reached.contains(stage) {
|
||||
report(
|
||||
Level::Fail,
|
||||
format!(
|
||||
"stage {stage:?} exists but {:?} is offered no moves that reach it",
|
||||
need.handled_by
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
for state in &need.success {
|
||||
if !need.done_when.contains(state) {
|
||||
report(
|
||||
Level::Fail,
|
||||
format!("success names {state:?}, which is not one of done_when"),
|
||||
);
|
||||
}
|
||||
}
|
||||
stranded.sort();
|
||||
for state in stranded {
|
||||
report(
|
||||
Level::Fail,
|
||||
format!(
|
||||
"records strand in {state:?}: {:?} is offered no move out of it",
|
||||
need.handled_by
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let need_ids: HashSet<&str> = file.needs.iter().map(|n| n.id.as_str()).collect();
|
||||
for persona in &file.personas {
|
||||
if !need_ids.contains(persona.need.as_str()) {
|
||||
findings.push(Finding {
|
||||
need: persona.need.clone(),
|
||||
level: Level::Fail,
|
||||
message: format!("persona {:?} points at a need that is not declared", persona.id),
|
||||
});
|
||||
}
|
||||
}
|
||||
findings
|
||||
}
|
||||
|
||||
/// Everything a simulation of the business needs, resolved: each
|
||||
/// need's path with the choice on every page and the form that gets
|
||||
/// filled, each bucket's state machine with the buttons every desk
|
||||
/// offers, and the personas. Portal resolves it because only portal
|
||||
/// loads content the way the runtime does (derived ids, relative
|
||||
/// refs, sections); whoever simulates reads plain JSON.
|
||||
pub fn sim_model(
|
||||
file: &NeedsFile,
|
||||
questions: &Questions,
|
||||
aggregates: &HashMap<String, AggregateSchema>,
|
||||
) -> serde_json::Value {
|
||||
use serde_json::json;
|
||||
let mut needs = Vec::new();
|
||||
let mut buckets = serde_json::Map::new();
|
||||
for need in &file.needs {
|
||||
let Some(path) = path_for(questions, need) else {
|
||||
continue;
|
||||
};
|
||||
let steps: Vec<serde_json::Value> = path
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, step)| {
|
||||
let page = &questions[&step.page];
|
||||
let mut trail: Vec<String> = path[..=i].iter().map(|s| s.page.clone()).collect();
|
||||
let expect: Vec<String> =
|
||||
viable(questions, need, page, need.max_steps - i, i == 0, false, &mut trail)
|
||||
.into_iter()
|
||||
.map(|v| v.alternative)
|
||||
.collect();
|
||||
let fields: Vec<serde_json::Value> = page
|
||||
.alternatives
|
||||
.iter()
|
||||
.find(|a| a.name == step.alternative)
|
||||
.into_iter()
|
||||
.flat_map(|a| &a.features)
|
||||
.flat_map(|f| &f.requirements)
|
||||
.map(|r| {
|
||||
json!({
|
||||
"name": r.name,
|
||||
"label": r.label.clone().unwrap_or_else(|| r.name.clone()),
|
||||
"type": r.kind,
|
||||
"optional": r.optional,
|
||||
"options": r.options,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
json!({
|
||||
"page": step.page,
|
||||
"question": page.name,
|
||||
"options": page.alternatives.iter().filter(|a| !a.disabled).map(|a| json!({
|
||||
"name": a.name,
|
||||
"description": plain(&a.description),
|
||||
})).collect::<Vec<_>>(),
|
||||
"expect": expect,
|
||||
"alternative": step.alternative,
|
||||
"fields": fields,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
needs.push(json!({
|
||||
"id": need.id,
|
||||
"who": need.who,
|
||||
"wants": need.wants,
|
||||
"enters_as": need.enters_as,
|
||||
"lands_in": need.lands_in,
|
||||
"handled_by": need.handled_by,
|
||||
"done_when": need.done_when,
|
||||
"success": need.success,
|
||||
"steps": steps,
|
||||
}));
|
||||
if let Some(schema) = aggregates.get(&need.lands_in) {
|
||||
buckets.entry(need.lands_in.clone()).or_insert_with(|| {
|
||||
json!({
|
||||
"initial": schema.initial,
|
||||
"transitions": schema.transitions,
|
||||
"desks": desks_over(questions, &need.lands_in).iter().map(|d| json!({
|
||||
"page": d.page,
|
||||
"group": d.group,
|
||||
"moves": d.moves.iter().map(|(from, to, label)| json!({
|
||||
"from": from, "to": to, "label": label,
|
||||
})).collect::<Vec<_>>(),
|
||||
})).collect::<Vec<_>>(),
|
||||
"self_moves": self_moves(questions, &need.lands_in),
|
||||
})
|
||||
});
|
||||
}
|
||||
}
|
||||
json!({
|
||||
"needs": needs,
|
||||
"buckets": buckets,
|
||||
"personas": file.personas.iter().map(|p| json!({
|
||||
"id": p.id, "need": p.need, "text": plain(&p.text),
|
||||
})).collect::<Vec<_>>(),
|
||||
})
|
||||
}
|
||||
|
||||
/// One decision for the answering engine - the shape is laya's
|
||||
/// `predict(state, questions)` input, so an adapter is a loop and
|
||||
/// nothing more; any other engine reads the same fields.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct Task {
|
||||
pub id: String,
|
||||
pub persona: String,
|
||||
pub need: String,
|
||||
pub page: String,
|
||||
pub state: BTreeMap<&'static str, String>,
|
||||
pub questions: BTreeMap<&'static str, ChoiceQuestion>,
|
||||
/// Every alternative that still meets the need from here. The
|
||||
/// engine never sees this.
|
||||
pub expect: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ChoiceQuestion {
|
||||
#[serde(rename = "type")]
|
||||
pub kind: &'static str,
|
||||
pub instructions: String,
|
||||
pub criteria: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
fn plain(text: &str) -> String {
|
||||
text.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
/// Every (persona, page-on-their-path) pair with a real choice on it.
|
||||
/// `skim` shows the engine only each alternative's heading - what a
|
||||
/// visitor who never reads the description has to go on.
|
||||
pub fn tasks(file: &NeedsFile, questions: &Questions, skim: bool) -> Vec<Task> {
|
||||
let mut out = Vec::new();
|
||||
for persona in &file.personas {
|
||||
let Some(need) = file.needs.iter().find(|n| n.id == persona.need) else {
|
||||
continue;
|
||||
};
|
||||
let Some(path) = path_for(questions, need) else {
|
||||
continue;
|
||||
};
|
||||
for (i, step) in path.iter().enumerate() {
|
||||
let page = &questions[&step.page];
|
||||
let open: Vec<&Alternative> = page.alternatives.iter().filter(|a| !a.disabled).collect();
|
||||
if open.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
let mut trail: Vec<String> = path[..=i].iter().map(|s| s.page.clone()).collect();
|
||||
let expect: Vec<String> = viable(questions, need, page, need.max_steps - i, i == 0, false, &mut trail)
|
||||
.into_iter()
|
||||
.map(|v| v.alternative)
|
||||
.collect();
|
||||
let criteria = open
|
||||
.iter()
|
||||
.map(|a| {
|
||||
let description = plain(&a.description);
|
||||
(
|
||||
a.name.clone(),
|
||||
if skim || description.is_empty() { a.name.clone() } else { description },
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
out.push(Task {
|
||||
id: format!("{}@{}", persona.id, step.page),
|
||||
persona: persona.id.clone(),
|
||||
need: need.id.clone(),
|
||||
page: step.page.clone(),
|
||||
state: BTreeMap::from([("body", plain(&persona.text))]),
|
||||
questions: BTreeMap::from([(
|
||||
"pick",
|
||||
ChoiceQuestion {
|
||||
kind: "choice",
|
||||
instructions: format!(
|
||||
"A visitor reads the page question {:?}. Which alternative do they choose?",
|
||||
page.name
|
||||
),
|
||||
criteria,
|
||||
},
|
||||
)]),
|
||||
expect,
|
||||
});
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct Answer {
|
||||
pub id: String,
|
||||
pub choice: String,
|
||||
#[serde(default)]
|
||||
pub confidence: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Score {
|
||||
pub total: usize,
|
||||
pub correct: usize,
|
||||
/// (task id, picked, expected, confidence)
|
||||
pub misroutes: Vec<(String, String, Vec<String>, Option<f64>)>,
|
||||
/// Right, but the engine was under `hesitant_below` - wording that
|
||||
/// works and is still close to a coin flip.
|
||||
pub hesitant: Vec<(String, f64)>,
|
||||
pub unanswered: Vec<String>,
|
||||
}
|
||||
|
||||
impl Score {
|
||||
pub fn accuracy(&self) -> f64 {
|
||||
if self.total == 0 {
|
||||
1.0
|
||||
} else {
|
||||
self.correct as f64 / self.total as f64
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn score(tasks: &[Task], answers: &[Answer], hesitant_below: f64) -> Score {
|
||||
let by_id: HashMap<&str, &Answer> = answers.iter().map(|a| (a.id.as_str(), a)).collect();
|
||||
let mut score = Score {
|
||||
total: tasks.len(),
|
||||
..Score::default()
|
||||
};
|
||||
for task in tasks {
|
||||
match by_id.get(task.id.as_str()) {
|
||||
None => score.unanswered.push(task.id.clone()),
|
||||
Some(answer) if task.expect.contains(&answer.choice) => {
|
||||
score.correct += 1;
|
||||
if let Some(c) = answer.confidence.filter(|c| *c < hesitant_below) {
|
||||
score.hesitant.push((task.id.clone(), c));
|
||||
}
|
||||
}
|
||||
Some(answer) => score.misroutes.push((
|
||||
task.id.clone(),
|
||||
answer.choice.clone(),
|
||||
task.expect.clone(),
|
||||
answer.confidence,
|
||||
)),
|
||||
}
|
||||
}
|
||||
score
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use portal::aggregates::parse_aggregates_yaml;
|
||||
use portal::content::build_questions;
|
||||
|
||||
fn site(desk_moves: &str) -> (Questions, HashMap<String, AggregateSchema>) {
|
||||
let files = vec![
|
||||
(
|
||||
"index.yaml".to_string(),
|
||||
"name: What are you here for?\nresponsible: { name: A, contact: a@b.c }\nalternatives:\n - name: Hire us\n description: Bring a problem.\n action: /thanks\n record_as: projects\n features:\n - name: Brief\n requirements:\n - name: email\n type: email\n - name: brief\n type: textarea\n - name: budget\n optional: true\n - name: Follow along\n description: Notes by mail.\n action: /thanks\n record_as: subscribers\n".to_string(),
|
||||
),
|
||||
("thanks.yaml".to_string(), "name: Thanks\nfollowup: true\n".to_string()),
|
||||
(
|
||||
"desk/index.yaml".to_string(),
|
||||
format!("name: Desk\nqualifies: owners\nalternatives:\n - name: Projects\n features:\n - name: \"\"\n resource:\n source: {{ kind: kv, bucket: projects }}\n requires_group: owners\n transitions:\n{desk_moves}"),
|
||||
),
|
||||
];
|
||||
let aggregates = parse_aggregates_yaml(
|
||||
"aggregates:\n - bucket: projects\n initial: open\n states:\n open: { event: submitted }\n quoted: { event: quoted }\n won: { event: won }\n lost: { event: lost }\n transitions:\n open: [quoted, lost]\n quoted: [won, lost]\n - bucket: subscribers\n attended_by: a mailer\n initial: open\n states:\n open: { event: subscribed }\n transitions: {}\n",
|
||||
)
|
||||
.unwrap();
|
||||
(build_questions(&files).unwrap(), aggregates)
|
||||
}
|
||||
|
||||
const FULL_DESK: &str = " - { to: quoted, label: Quote }\n - { to: lost, label: Lose }\n - { from: quoted, to: won, label: Win }\n - { from: quoted, to: lost, label: Lose }\n";
|
||||
|
||||
fn needs(extra: &str) -> NeedsFile {
|
||||
parse(&format!(
|
||||
"needs:\n - id: hire\n who: a client\n wants: work done\n lands_in: projects\n handled_by: owners\n done_when: [won, lost]\n{extra}personas:\n - id: founder\n need: hire\n text: I need someone to build our booking tool.\n"
|
||||
))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn fails(findings: &[Finding]) -> Vec<&str> {
|
||||
findings
|
||||
.iter()
|
||||
.filter(|f| f.level == Level::Fail)
|
||||
.map(|f| f.message.as_str())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_met_need_is_silent() {
|
||||
let (questions, aggregates) = site(FULL_DESK);
|
||||
let findings = check(&needs(" max_fields: 2\n"), &questions, &aggregates);
|
||||
assert!(findings.is_empty(), "{findings:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effort_budget_counts_required_fields_only() {
|
||||
let (questions, aggregates) = site(FULL_DESK);
|
||||
let findings = check(&needs(" max_fields: 1\n"), &questions, &aggregates);
|
||||
assert_eq!(fails(&findings), vec!["the path asks for 2 required field(s), over the budget of 1"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_desk_missing_a_move_strands_records_and_cannot_finish() {
|
||||
// The desk can quote, but never close a quoted job.
|
||||
let (questions, aggregates) = site(" - { to: quoted, label: Quote }\n - { to: lost, label: Lose }\n");
|
||||
let findings = check(&needs(""), &questions, &aggregates);
|
||||
assert_eq!(
|
||||
fails(&findings),
|
||||
vec![
|
||||
"\"owners\" cannot move a record to \"won\" - no desk of theirs offers the moves",
|
||||
"records strand in \"quoted\": \"owners\" is offered no move out of it",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_finished_state_may_lead_on_to_another() {
|
||||
// quoted counts as handled here, yet won must still be
|
||||
// reachable past it; lost-after-quoted is not this need's
|
||||
// business, so nothing strands.
|
||||
let (questions, aggregates) = site(FULL_DESK);
|
||||
let mut file = needs("");
|
||||
file.needs[0].done_when = vec!["quoted".to_string(), "won".to_string(), "lost".to_string()];
|
||||
let findings = check(&file, &questions, &aggregates);
|
||||
assert!(findings.is_empty(), "{findings:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_disabled_alternative_is_named_as_the_reason() {
|
||||
let (mut questions, aggregates) = site(FULL_DESK);
|
||||
questions.get_mut("/").unwrap().alternatives[0].disabled = true;
|
||||
let findings = check(&needs(""), &questions, &aggregates);
|
||||
assert_eq!(
|
||||
fails(&findings),
|
||||
vec!["the path exists but is closed: \"Hire us\" on / is `disabled`, so the answer is advertised and refused"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn success_must_be_a_finished_state_and_the_sim_model_carries_the_whole_flow() {
|
||||
let (questions, aggregates) = site(FULL_DESK);
|
||||
let mut file = needs(" success: [won]\n");
|
||||
assert!(check(&file, &questions, &aggregates).is_empty());
|
||||
|
||||
let model = sim_model(&file, &questions, &aggregates);
|
||||
let need = &model["needs"][0];
|
||||
assert_eq!(need["success"][0], "won");
|
||||
let step = &need["steps"][0];
|
||||
assert_eq!(step["question"], "What are you here for?");
|
||||
assert_eq!(step["expect"][0], "Hire us");
|
||||
assert_eq!(step["options"].as_array().unwrap().len(), 2);
|
||||
// The form the visitor fills, required and optional alike.
|
||||
let fields = step["fields"].as_array().unwrap();
|
||||
assert_eq!(fields.len(), 3);
|
||||
assert_eq!(fields[2]["optional"], true);
|
||||
// The desk's buttons, labelled, per state.
|
||||
let moves = model["buckets"]["projects"]["desks"][0]["moves"].as_array().unwrap();
|
||||
assert!(moves.iter().any(|m| m["from"] == "quoted" && m["to"] == "won" && m["label"] == "Win"));
|
||||
assert_eq!(model["buckets"]["projects"]["initial"], "open");
|
||||
assert_eq!(model["personas"][0]["id"], "founder");
|
||||
|
||||
// A stage the business named must exist as a state.
|
||||
file.needs[0].stages = vec!["open".into(), "surveyed".into(), "quoted".into()];
|
||||
assert_eq!(
|
||||
fails(&check(&file, &questions, &aggregates)),
|
||||
vec!["the business describes a stage \"surveyed\" that \"projects\" has no state for"]
|
||||
);
|
||||
file.needs[0].stages = vec!["open".into(), "quoted".into(), "won".into()];
|
||||
assert!(check(&file, &questions, &aggregates).is_empty());
|
||||
|
||||
file.needs[0].success = vec!["quoted".to_string()];
|
||||
assert_eq!(
|
||||
fails(&check(&file, &questions, &aggregates)),
|
||||
vec!["success names \"quoted\", which is not one of done_when"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_need_can_be_carried_by_more_than_one_groups_desk() {
|
||||
// owners may only quote or lose; closing a quoted job is the
|
||||
// site managers' button, on a desk of their own.
|
||||
let (mut questions, aggregates) = site(" - { to: quoted, label: Quote }\n - { to: lost, label: Lose }\n");
|
||||
let more = build_questions(&[(
|
||||
"site/index.yaml".to_string(),
|
||||
"name: Site desk\nqualifies: site_managers\nalternatives:\n - name: Projects\n features:\n - name: \"\"\n resource:\n source: { kind: kv, bucket: projects }\n requires_group: site_managers\n transitions:\n - { from: quoted, to: won, label: Win }\n - { from: quoted, to: lost, label: Lose }\n".to_string(),
|
||||
)])
|
||||
.unwrap();
|
||||
questions.extend(more);
|
||||
|
||||
let mut file = needs("");
|
||||
// Alone, owners cannot finish it...
|
||||
assert_eq!(fails(&check(&file, &questions, &aggregates)).len(), 2);
|
||||
// ...with the site managers' buttons counted, they can.
|
||||
file.needs[0].also_moved_by = vec!["site_managers".to_string()];
|
||||
assert!(check(&file, &questions, &aggregates).is_empty());
|
||||
// But someone has to be answerable: handled_by needs its own desk.
|
||||
file.needs[0].handled_by = "nobody".to_string();
|
||||
assert!(fails(&check(&file, &questions, &aggregates))[0].starts_with("no desk gated on \"nobody\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_wrong_group_holding_the_desk_is_a_failure() {
|
||||
let (questions, aggregates) = site(FULL_DESK);
|
||||
let mut file = needs("");
|
||||
file.needs[0].handled_by = "sales".to_string();
|
||||
let findings = check(&file, &questions, &aggregates);
|
||||
assert_eq!(
|
||||
fails(&findings),
|
||||
vec!["no desk gated on \"sales\" reads \"projects\" - it is read by /desk (owners)"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn via_pins_the_entry_alternative() {
|
||||
let (questions, aggregates) = site(FULL_DESK);
|
||||
let mut file = needs("");
|
||||
file.needs[0].via = Some("Follow along".to_string());
|
||||
let findings = check(&file, &questions, &aggregates);
|
||||
assert_eq!(
|
||||
fails(&findings),
|
||||
vec!["no path from \"/\" via \"Follow along\" records into \"projects\" within 2 submission(s)"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tasks_carry_the_page_as_a_choice_and_score_separates_misroutes() {
|
||||
let (questions, _) = site(FULL_DESK);
|
||||
let file = needs("");
|
||||
let tasks = tasks(&file, &questions, false);
|
||||
let skimmed = super::tasks(&file, &questions, true);
|
||||
assert_eq!(skimmed[0].questions["pick"].criteria["Follow along"], "Follow along");
|
||||
assert_eq!(tasks.len(), 1);
|
||||
let task = &tasks[0];
|
||||
assert_eq!(task.id, "founder@/");
|
||||
assert_eq!(task.expect, vec!["Hire us"]);
|
||||
let q = &task.questions["pick"];
|
||||
assert_eq!(q.kind, "choice");
|
||||
assert_eq!(q.criteria["Follow along"], "Notes by mail.");
|
||||
assert!(q.instructions.contains("What are you here for?"));
|
||||
|
||||
let right = vec![Answer { id: "founder@/".into(), choice: "Hire us".into(), confidence: Some(0.55) }];
|
||||
let s = score(&tasks, &right, 0.6);
|
||||
assert_eq!((s.correct, s.misroutes.len(), s.hesitant.len()), (1, 0, 1));
|
||||
|
||||
let wrong = vec![Answer { id: "founder@/".into(), choice: "Follow along".into(), confidence: Some(0.9) }];
|
||||
let s = score(&tasks, &wrong, 0.6);
|
||||
assert_eq!((s.correct, s.misroutes.len()), (0, 1));
|
||||
assert_eq!(s.accuracy(), 0.0);
|
||||
|
||||
assert_eq!(score(&tasks, &[], 0.6).unanswered, vec!["founder@/"]);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user