Bring the lint in (uhhm/iris main, history kept)
git-subtree-dir: vendor/lint git-subtree-mainline:bedd306854git-subtree-split:e3c68f9441
This commit is contained in:
Bendik Aagaard Lynghaug
+46
@@ -0,0 +1,46 @@
|
||||
name: Publish release
|
||||
|
||||
# A `v*` tag builds the binary and attaches it to the Gitea release as
|
||||
# `iris`, with the tag's CHANGELOG.md section as the notes. Content
|
||||
# repos pin IRIS_RELEASE to one of these tags in their lint workflow.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ["v*"]
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: bare
|
||||
env:
|
||||
# Writable, unlike the shared /var/local/cargo - see test.yml.
|
||||
CARGO_HOME: /var/local/cargo-target/iris-cargo-home
|
||||
RUSTC_WRAPPER: /usr/bin/sccache
|
||||
RUSTUP_HOME: /var/local/rustup
|
||||
PATH: /var/local/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/bin
|
||||
SCCACHE_DIR: /var/local/sccache
|
||||
SCCACHE_SERVER_PORT: "4228"
|
||||
CARGO_TARGET_DIR: /var/local/cargo-target
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Build
|
||||
run: cargo build --release
|
||||
|
||||
- name: Publish release
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tag="${{ github.ref_name }}"
|
||||
api="${{ github.server_url }}/api/v1/repos/${{ github.repository }}"
|
||||
auth="Authorization: token ${{ secrets.GITHUB_TOKEN }}"
|
||||
notes=$(awk -v v="${tag#v}" '/^## /{p = ($2 == v)} p' CHANGELOG.md | tail -n +2)
|
||||
body=$(jq -n --arg tag "$tag" --arg notes "$notes" '{tag_name: $tag, name: $tag, body: $notes}')
|
||||
id=$(curl -sf -X POST -H "$auth" -H 'Content-Type: application/json' \
|
||||
-d "$body" "$api/releases" | jq .id) \
|
||||
|| id=$(curl -sf -H "$auth" "$api/releases/tags/$tag" | jq .id)
|
||||
for aid in $(curl -sf -H "$auth" "$api/releases/$id/assets" | jq '.[].id'); do
|
||||
curl -sf -X DELETE -H "$auth" "$api/releases/$id/assets/$aid"
|
||||
done
|
||||
curl -sf -X POST -H "$auth" \
|
||||
-F "attachment=@/var/local/cargo-target/release/iris" \
|
||||
"$api/releases/$id/assets?name=iris" > /dev/null
|
||||
echo "published $tag"
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
name: Test
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: bare
|
||||
env:
|
||||
# Same shared toolchain and cache as uhhm/portal's workflows.
|
||||
# Not the shared /var/local/cargo: the runner may only read it, and
|
||||
# the portal git dependency has to be cloned into CARGO_HOME. This
|
||||
# one lives under the writable target dir; the rustc wrapper the
|
||||
# shared config would have set is given here instead.
|
||||
CARGO_HOME: /var/local/cargo-target/iris-cargo-home
|
||||
RUSTC_WRAPPER: /usr/bin/sccache
|
||||
RUSTUP_HOME: /var/local/rustup
|
||||
PATH: /var/local/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/bin
|
||||
SCCACHE_DIR: /var/local/sccache
|
||||
SCCACHE_SERVER_PORT: "4228"
|
||||
CARGO_TARGET_DIR: /var/local/cargo-target
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Test
|
||||
run: cargo test
|
||||
|
||||
# The one thing a lint must never do is reject the content it
|
||||
# is meant to serve: every live content repo has to pass.
|
||||
- name: Check the live content repos
|
||||
run: |
|
||||
cargo build --release
|
||||
for repo in uhhm/questions redoal/questions; do
|
||||
/var/local/cargo-target/release/iris check --repo "${{ github.server_url }}/$repo"
|
||||
done
|
||||
@@ -0,0 +1 @@
|
||||
target/
|
||||
Vendored
+104
@@ -0,0 +1,104 @@
|
||||
# Changelog
|
||||
|
||||
One line per change, grouped by the release that shipped it. Newest
|
||||
first. Each release names the portal tag it is built against.
|
||||
|
||||
## Unreleased
|
||||
|
||||
- `needs.yaml` may name the site's `tones`, one storyline each; the checks read the first storyline, which is what `build_questions` gives.
|
||||
|
||||
## 0.5.10 (2026-09-30) - portal v0.5.10
|
||||
|
||||
- Matches portal v0.5.10 (released by portal's publish job).
|
||||
|
||||
## 0.5.9 (2026-09-30) - portal v0.5.9
|
||||
|
||||
- Matches portal v0.5.9 (released by portal's publish job).
|
||||
|
||||
## 0.5.8 (2026-09-30) - portal v0.5.8
|
||||
|
||||
- Matches portal v0.5.8 (released by portal's publish job).
|
||||
|
||||
## 0.5.7 (2026-09-30) - portal v0.5.7
|
||||
|
||||
- Matches portal v0.5.7 (released by portal's publish job).
|
||||
|
||||
## Unreleased
|
||||
|
||||
- The effort budget counts what a person fills in: a `type: record` field (the case an answer is about, carried on the link) and a `hidden` preset are not asked and no longer count. An objection page whose form carried the application's id used to be one field over.
|
||||
|
||||
## 0.5.6 (2026-09-30) - portal v0.5.6
|
||||
|
||||
- Matches portal v0.5.6 (released by portal's publish job).
|
||||
|
||||
## 0.5.5 (2026-09-30) - portal v0.5.5
|
||||
|
||||
- Matches portal v0.5.5 (released by portal's publish job).
|
||||
|
||||
## 0.5.4 (2026-09-29) - portal v0.5.4
|
||||
|
||||
- Matches portal v0.5.4 (released by portal's publish job).
|
||||
|
||||
## 0.5.3 (2026-09-29) - portal v0.5.3
|
||||
|
||||
- Matches portal v0.5.3 (released by portal's publish job).
|
||||
|
||||
## 0.5.2 (2026-09-28) - portal v0.5.2
|
||||
|
||||
- Matches portal v0.5.2 (released by portal's publish job).
|
||||
|
||||
## 0.5.1 (2026-09-28) - portal v0.5.1
|
||||
|
||||
- Matches portal v0.5.1 (released by portal's publish job).
|
||||
|
||||
## 0.5.0 (2026-09-28) - portal v0.5.0
|
||||
|
||||
- Matches portal v0.5.0 (released by portal's publish job).
|
||||
|
||||
## 0.4.0 (2026-09-24) - portal v0.4.0
|
||||
|
||||
- Matches portal v0.4.0 (released by portal's publish job).
|
||||
|
||||
## 0.3.47 (2026-09-24) - portal v0.3.47
|
||||
|
||||
- Matches portal v0.3.47 (released by portal's publish job).
|
||||
|
||||
## 0.3.46 (2026-09-23) - portal v0.3.46
|
||||
|
||||
- Matches portal v0.3.46 (released by portal's publish job).
|
||||
|
||||
## 0.3.45 (2026-09-23) - portal v0.3.45
|
||||
|
||||
- Matches portal v0.3.45 (released by portal's publish job).
|
||||
|
||||
## 0.3.44 (2026-09-23) - portal v0.3.44
|
||||
|
||||
- Matches portal v0.3.44 (released by portal's publish job).
|
||||
|
||||
## 0.3.43 (2026-09-23) - portal v0.3.43
|
||||
|
||||
- Matches portal v0.3.43 (released by portal's publish job).
|
||||
|
||||
## 0.3.42 (2026-09-23) - portal v0.3.42
|
||||
|
||||
- Matches portal v0.3.42 (released by portal's publish job).
|
||||
|
||||
## 0.3.41 (2026-09-23) - portal v0.3.41
|
||||
|
||||
- Matches portal v0.3.41 (released by portal's publish job).
|
||||
|
||||
## 0.3.39 (2026-09-23) - portal v0.3.39
|
||||
|
||||
- Matches portal v0.3.39 (released by portal's publish job).
|
||||
|
||||
## 0.1.1 (2026-09-22) - portal v0.3.36
|
||||
|
||||
- `check --repo` fetches needs.yaml itself; the v0.1.0 tag never published (a cache permission on the runner, since fixed).
|
||||
|
||||
## 0.1.0 (2026-09-22) - portal v0.3.36
|
||||
|
||||
- First release: the lint moved out of portal (`question_lint` becomes
|
||||
`iris check`, `needs_replay` becomes `iris replay`), with the
|
||||
business-needs pass, the wording task export and scoring, and the
|
||||
simulation model export. Portal is a library dependency pinned to
|
||||
the release iris matches.
|
||||
+5991
File diff suppressed because it is too large
Load Diff
Vendored
+25
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "iris"
|
||||
version = "0.5.10"
|
||||
edition = "2021"
|
||||
description = "The iris over a portal site: nothing from a content repo comes through until it checks out"
|
||||
license = "MIT"
|
||||
repository = "https://project.uhhm.no/uhhm/iris"
|
||||
|
||||
[dependencies]
|
||||
# The site's own content types and loaders, so iris reads a repo exactly
|
||||
# the way the running site does. Pinned to the portal release it
|
||||
# matches; bumping this tag is what a new iris release is.
|
||||
portal = { git = "https://project.uhhm.no/uhhm/portal.git", tag = "v0.5.10", features = ["ssr"] }
|
||||
anyhow = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
serde_yaml = "0.9"
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
|
||||
async-nats = "0.38"
|
||||
futures = "0.3"
|
||||
chrono = "0.4"
|
||||
|
||||
[profile.release]
|
||||
strip = true
|
||||
lto = "thin"
|
||||
Vendored
+57
@@ -0,0 +1,57 @@
|
||||
# iris
|
||||
|
||||
The iris over a [portal](https://project.uhhm.no/uhhm/portal) site. A
|
||||
content push is an incoming wormhole; nothing comes through until it
|
||||
checks out.
|
||||
|
||||
```
|
||||
iris check --path questions # a local checkout
|
||||
iris check --repo https://project.uhhm.no/uhhm/questions
|
||||
iris --path questions # the same, for one-line CI
|
||||
```
|
||||
|
||||
`check` loads a content repo exactly the way the running site does,
|
||||
validates it (every action lands on a page, every transition is one
|
||||
the state machine allows, every recorded bucket is read back
|
||||
somewhere), and then holds it to the business needs the repo declares
|
||||
in `needs.yaml`, when it has one:
|
||||
|
||||
- a path of submissions gets each kind of visitor's answer into the
|
||||
bucket it names, within a submission and required-field budget;
|
||||
- the handling group holds a desk over that bucket, and every
|
||||
finished state is reachable through the buttons it (and any
|
||||
`also_moved_by` group) offers; no record strands on the way;
|
||||
- every stage the business named is a state of the machine.
|
||||
|
||||
A need marked `planned` warns instead of failing. Exit status is the
|
||||
verdict, so a content repo's CI is one line.
|
||||
|
||||
Beyond the verdict, `check` exports what a trainer needs to grade the
|
||||
wording: `--needs-tasks` writes the personas as typed choice tasks
|
||||
(`--skim` for headings only), `--needs-score` grades an engine's
|
||||
answers, and `--needs-sim` writes a resolved model of the whole site.
|
||||
`iris replay` runs simulated cases through the site's real aggregate
|
||||
engine on a throwaway JetStream and reports every bucket by state;
|
||||
`--report-only` takes the same scorecard from a live site's buckets.
|
||||
See [portal-trainer](https://project.uhhm.no/uhhm/portal-trainer) for
|
||||
what sits around those.
|
||||
|
||||
## Matching the site
|
||||
|
||||
Every type iris reads is portal's own: portal is a library dependency
|
||||
pinned to a release tag in `Cargo.toml`, and iris carries the same
|
||||
version number as that tag, so `iris v0.3.39` is the lint for portal
|
||||
v0.3.39 and nothing else. A content repo pins `IRIS_RELEASE` to the
|
||||
`PORTAL_RELEASE` its site runs. Portal's own release tarball ships the
|
||||
matching `iris` binary too, at `/srv/app/<instance>/current/iris`.
|
||||
|
||||
## Release
|
||||
|
||||
Iris is released by portal's publish job, never by hand: when a
|
||||
portal tag is pushed, that job re-pins this crate to the tag, sets
|
||||
the version to match, builds and tests it, adds the changelog line,
|
||||
and pushes the commit and the tag here. This repo's own CI then
|
||||
attaches the binary to the Gitea release as `iris`. A portal change
|
||||
to something the lint reads fails the portal release at that step.
|
||||
Changes to iris itself land on `main` between releases and ship with
|
||||
the next portal tag.
|
||||
Vendored
+246
@@ -0,0 +1,246 @@
|
||||
//! Headless schema-check binary - loads content and `aggregates.yaml`
|
||||
//! (from a Gitea repo URL or a local directory) and validates them
|
||||
//! exactly the way `content::watch_for_reload`/`main.rs`'s boot path
|
||||
//! do, with no NATS, OIDC, web server, or JetStream connection
|
||||
//! involved. Built once by portal's own deploy workflow and run
|
||||
//! directly by `questions`' own CI (same bare-metal runner/host,
|
||||
//! published to a stable path - no artifact download needed), rather
|
||||
//! than compiled there - keeps that repo's CI coupling to "run a
|
||||
//! static binary," not "build a Rust workspace."
|
||||
|
||||
use crate::local::{load_aggregates_from_dir, load_from_dir, load_needs_from_dir, load_site_from_dir};
|
||||
use crate::needs;
|
||||
use portal::{aggregates, content};
|
||||
|
||||
pub async fn run(argv: Vec<String>) -> anyhow::Result<()> {
|
||||
let mut args = argv.into_iter();
|
||||
let mut repo: Option<String> = None;
|
||||
let mut branch = "main".to_string();
|
||||
let mut subdir = "questions".to_string();
|
||||
let mut path: Option<String> = None;
|
||||
let mut tasks_out: Option<String> = None;
|
||||
let mut answers_in: Option<String> = None;
|
||||
let mut min_accuracy: f64 = 0.0;
|
||||
let mut skim = false;
|
||||
let mut sim_out: Option<String> = None;
|
||||
let mut show_access = false;
|
||||
|
||||
while let Some(arg) = args.next() {
|
||||
match arg.as_str() {
|
||||
"--repo" => repo = args.next(),
|
||||
"--branch" => branch = args.next().unwrap_or(branch),
|
||||
"--subdir" => subdir = args.next().unwrap_or(subdir),
|
||||
"--path" => path = args.next(),
|
||||
"--needs-tasks" => tasks_out = args.next(),
|
||||
"--skim" => skim = true,
|
||||
"--needs-sim" => sim_out = args.next(),
|
||||
"--access" => show_access = true,
|
||||
"--needs-score" => answers_in = args.next(),
|
||||
"--min-accuracy" => {
|
||||
min_accuracy = args.next().and_then(|v| v.parse().ok()).unwrap_or(min_accuracy)
|
||||
}
|
||||
other => {
|
||||
eprintln!("unknown argument: {other}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let (questions, aggregates_map, site, needs_raw) = if let Some(dir) = path {
|
||||
(
|
||||
load_from_dir(&dir)?,
|
||||
load_aggregates_from_dir(&dir)?,
|
||||
load_site_from_dir(&dir)?,
|
||||
load_needs_from_dir(&dir)?,
|
||||
)
|
||||
} else if let Some(repo_url) = repo {
|
||||
let questions = content::load_questions_from_gitea(&repo_url, &branch, &subdir).await?;
|
||||
let aggregates_map = content::load_aggregates_from_gitea(&repo_url, &branch).await?;
|
||||
// load_site_from_gitea already validates; a missing file is
|
||||
// the default config, same as at portal boot.
|
||||
let site = content::load_site_from_gitea(&repo_url, &branch).await?;
|
||||
// Optional: a fetch failure means the repo declares no needs.
|
||||
let needs_raw = portal::http::get_text(&format!("{}/needs.yaml?ref={branch}", content::gitea_raw_base(&repo_url)?), portal::http::GITEA_CAP).await.ok();
|
||||
(questions, aggregates_map, site, needs_raw)
|
||||
} else {
|
||||
eprintln!(
|
||||
"usage: iris check (--repo <gitea-url> [--branch main] [--subdir questions] | --path <local-dir>) [--needs-tasks <out.jsonl> [--skim]] [--needs-sim <model.json>] [--access] [--needs-score <answers.jsonl> [--min-accuracy 0.8]]"
|
||||
);
|
||||
std::process::exit(2);
|
||||
};
|
||||
|
||||
if let Err(e) = site.validate() {
|
||||
eprintln!("FAIL: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
let aggregates_map = content::with_builtin_aggregates(aggregates_map);
|
||||
match content::validate_questions(&questions, &aggregates_map) {
|
||||
Ok(()) => {
|
||||
// The runtime's own event desk: a repo announcing pages
|
||||
// (Question.event) should read portal_events somewhere, or
|
||||
// "post what happened" tasks pile up unseen. Warn, don't
|
||||
// fail - the runtime creates the records either way.
|
||||
if questions.values().any(|q| q.event.is_some())
|
||||
&& !content::bucket_is_read(&questions, content::EVENTS_BUCKET)
|
||||
{
|
||||
eprintln!(
|
||||
"WARN: pages carry `event:` but no kv resource reads bucket {:?} - add a desk so summaries get posted",
|
||||
content::EVENTS_BUCKET
|
||||
);
|
||||
}
|
||||
println!(
|
||||
"OK: {} question(s), {} aggregate(s) valid",
|
||||
questions.len(),
|
||||
aggregates_map.len()
|
||||
);
|
||||
// The access policy the site will compile from this content.
|
||||
// It must type-check against portal's schema: a rule the
|
||||
// schema cannot express would grant nothing at runtime.
|
||||
let policy = portal::access::Policy::from_content(&questions, &aggregates_map);
|
||||
if let Err(e) = policy.validate() {
|
||||
eprintln!("FAIL: access policy does not validate: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
println!("OK: access policy, {} rule(s), validates", policy.rules.len());
|
||||
if show_access {
|
||||
println!("\n{:<16} {:<44} {:<40} {}", "ACTION", "RESOURCE", "WHO", "WHEN");
|
||||
let mut rows = policy.rules.clone();
|
||||
rows.sort_by(|a, b| (&a.resource, &a.action, &a.who).cmp(&(&b.resource, &b.action, &b.who)));
|
||||
for r in rows {
|
||||
println!("{:<16} {:<44} {:<40} {}", r.action, r.resource, r.who, r.when);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
check_needs(
|
||||
needs_raw.as_deref(),
|
||||
&questions,
|
||||
&aggregates_map,
|
||||
tasks_out.as_deref(),
|
||||
skim,
|
||||
sim_out.as_deref(),
|
||||
answers_in.as_deref(),
|
||||
min_accuracy,
|
||||
)
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("FAIL: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The business-needs pass (see `crate::needs`): structure always,
|
||||
/// then the wording tasks written out and/or an engine's answers
|
||||
/// scored, when asked. A repo without `needs.yaml` skips all of it.
|
||||
fn check_needs(
|
||||
raw: Option<&str>,
|
||||
questions: &std::collections::HashMap<String, content::Question>,
|
||||
aggregates_map: &std::collections::HashMap<String, aggregates::AggregateSchema>,
|
||||
tasks_out: Option<&str>,
|
||||
skim: bool,
|
||||
sim_out: Option<&str>,
|
||||
answers_in: Option<&str>,
|
||||
min_accuracy: f64,
|
||||
) -> anyhow::Result<()> {
|
||||
let Some(raw) = raw else {
|
||||
if tasks_out.is_some() || answers_in.is_some() {
|
||||
eprintln!("FAIL: --needs-tasks/--needs-score given, but the repo has no needs.yaml");
|
||||
std::process::exit(1);
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
let file = match needs::parse(raw) {
|
||||
Ok(file) => file,
|
||||
Err(e) => {
|
||||
eprintln!("FAIL: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
let findings = needs::check(&file, questions, aggregates_map);
|
||||
let mut failed = 0;
|
||||
for finding in &findings {
|
||||
let level = match finding.level {
|
||||
needs::Level::Fail => {
|
||||
failed += 1;
|
||||
"FAIL"
|
||||
}
|
||||
needs::Level::Warn => "WARN",
|
||||
};
|
||||
eprintln!("{level}: need {:?}: {}", finding.need, finding.message);
|
||||
}
|
||||
for need in &file.needs {
|
||||
if let Some(path) = needs::path_for(questions, need) {
|
||||
let steps: Vec<String> = path
|
||||
.iter()
|
||||
.map(|s| format!("{} [{}]", s.page, s.alternative))
|
||||
.collect();
|
||||
println!("need {:?}: {} -> {:?}", need.id, steps.join(" -> "), need.lands_in);
|
||||
}
|
||||
}
|
||||
if failed == 0 {
|
||||
let planned = file.needs.iter().filter(|n| n.planned).count();
|
||||
println!(
|
||||
"OK: {} need(s) structurally met, {} planned, {} persona(s)",
|
||||
file.needs.len() - planned,
|
||||
planned,
|
||||
file.personas.len()
|
||||
);
|
||||
}
|
||||
// The wording pass still runs for the needs that do have a path -
|
||||
// a closed flow elsewhere is no reason to stop grading the rest.
|
||||
|
||||
if let Some(out) = sim_out {
|
||||
let model = needs::sim_model(&file, questions, aggregates_map);
|
||||
std::fs::write(out, serde_json::to_string_pretty(&model)?)
|
||||
.map_err(|e| anyhow::anyhow!("writing {out}: {e}"))?;
|
||||
println!("wrote the simulation model to {out}");
|
||||
}
|
||||
let tasks = needs::tasks(&file, questions, skim);
|
||||
if let Some(out) = tasks_out {
|
||||
let mut lines = String::new();
|
||||
for task in &tasks {
|
||||
lines.push_str(&serde_json::to_string(task)?);
|
||||
lines.push('\n');
|
||||
}
|
||||
std::fs::write(out, lines).map_err(|e| anyhow::anyhow!("writing {out}: {e}"))?;
|
||||
println!("wrote {} wording task(s) to {out}", tasks.len());
|
||||
}
|
||||
if let Some(path) = answers_in {
|
||||
let raw = std::fs::read_to_string(path).map_err(|e| anyhow::anyhow!("reading {path}: {e}"))?;
|
||||
let mut answers = Vec::new();
|
||||
for (n, line) in raw.lines().enumerate().filter(|(_, l)| !l.trim().is_empty()) {
|
||||
answers.push(
|
||||
serde_json::from_str::<needs::Answer>(line)
|
||||
.map_err(|e| anyhow::anyhow!("{path}:{}: {e}", n + 1))?,
|
||||
);
|
||||
}
|
||||
let score = needs::score(&tasks, &answers, 0.6);
|
||||
for (id, picked, expected, confidence) in &score.misroutes {
|
||||
eprintln!(
|
||||
"MISROUTE: {id}: picked {picked:?}{}, the need is met by {expected:?}",
|
||||
confidence.map(|c| format!(" ({c:.2})")).unwrap_or_default()
|
||||
);
|
||||
}
|
||||
for (id, confidence) in &score.hesitant {
|
||||
eprintln!("HESITANT: {id}: right, at {confidence:.2}");
|
||||
}
|
||||
for id in &score.unanswered {
|
||||
eprintln!("UNANSWERED: {id}");
|
||||
}
|
||||
println!(
|
||||
"wording: {}/{} routed right ({:.0}%)",
|
||||
score.correct,
|
||||
score.total,
|
||||
score.accuracy() * 100.0
|
||||
);
|
||||
if score.accuracy() < min_accuracy {
|
||||
eprintln!("FAIL: below --min-accuracy {min_accuracy}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
eprintln!("FAIL: {failed} unmet business need finding(s)");
|
||||
std::process::exit(1);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Vendored
+99
@@ -0,0 +1,99 @@
|
||||
//! Loading a content repo from a local checkout - the offline
|
||||
//! counterparts to the Gitea loaders in `content`, for tools that lint
|
||||
//! or replay a branch that has not been pushed (question_lint,
|
||||
//! needs_replay). `dir` is always the pages directory (`questions/`);
|
||||
//! `site.yaml`, `aggregates.yaml` and `needs.yaml` sit one level up.
|
||||
|
||||
use portal::{aggregates, content};
|
||||
|
||||
/// The offline counterpart to `content::load_site_from_gitea` -
|
||||
/// `site.yaml` lives at the repo root like `aggregates.yaml`, and is
|
||||
/// just as optional locally: missing means default branding.
|
||||
pub fn load_site_from_dir(dir: &str) -> anyhow::Result<content::SiteConfig> {
|
||||
let site_path = std::path::Path::new(dir)
|
||||
.parent()
|
||||
.unwrap_or_else(|| std::path::Path::new("."))
|
||||
.join("site.yaml");
|
||||
if !site_path.exists() {
|
||||
return Ok(content::SiteConfig::default());
|
||||
}
|
||||
let raw = std::fs::read_to_string(&site_path)
|
||||
.map_err(|e| anyhow::anyhow!("reading {}: {e}", site_path.display()))?;
|
||||
serde_yaml::from_str(&raw).map_err(|e| anyhow::anyhow!("parsing {}: {e}", site_path.display()))
|
||||
}
|
||||
|
||||
/// The offline counterpart to `content::load_aggregates_from_gitea` -
|
||||
/// `aggregates.yaml` lives at the repo root, one level up from the
|
||||
/// pages directory `--path` names, so `dir`'s parent is where it's
|
||||
/// looked for. Missing entirely is not an error here (unlike
|
||||
/// `--repo` mode, where `aggregates.yaml` is required) - a local
|
||||
/// checkout being linted may not have one, and every declared
|
||||
/// transition still gets checked against whatever *is* found; an
|
||||
/// empty map just means nothing is checked.
|
||||
pub fn load_aggregates_from_dir(
|
||||
dir: &str,
|
||||
) -> anyhow::Result<std::collections::HashMap<String, aggregates::AggregateSchema>> {
|
||||
let aggregates_path = std::path::Path::new(dir)
|
||||
.parent()
|
||||
.unwrap_or_else(|| std::path::Path::new("."))
|
||||
.join("aggregates.yaml");
|
||||
if !aggregates_path.exists() {
|
||||
return Ok(std::collections::HashMap::new());
|
||||
}
|
||||
let raw = std::fs::read_to_string(&aggregates_path)
|
||||
.map_err(|e| anyhow::anyhow!("reading {}: {e}", aggregates_path.display()))?;
|
||||
aggregates::parse_aggregates_yaml(&raw)
|
||||
}
|
||||
|
||||
/// The offline counterpart to `content::load_questions_from_gitea` -
|
||||
/// the same recursive tree walk and `content::build_questions`
|
||||
/// pipeline (derived ids, relative refs, sections, followup
|
||||
/// inference), just reading a local checkout instead of Gitea's API,
|
||||
/// for linting a branch that hasn't been pushed yet.
|
||||
pub fn load_from_dir(
|
||||
dir: &str,
|
||||
) -> anyhow::Result<std::collections::HashMap<String, content::Question>> {
|
||||
let base = std::path::Path::new(dir);
|
||||
let mut files = Vec::new();
|
||||
collect_yaml(base, base, &mut files)?;
|
||||
content::build_questions(&files)
|
||||
}
|
||||
|
||||
fn collect_yaml(
|
||||
base: &std::path::Path,
|
||||
dir: &std::path::Path,
|
||||
out: &mut Vec<(String, String)>,
|
||||
) -> anyhow::Result<()> {
|
||||
for entry in std::fs::read_dir(dir)? {
|
||||
let path = entry?.path();
|
||||
if path.is_dir() {
|
||||
collect_yaml(base, &path, out)?;
|
||||
} else if path.extension().and_then(|e| e.to_str()) == Some("yaml") {
|
||||
let rel = path
|
||||
.strip_prefix(base)
|
||||
.expect("walked paths sit under their base")
|
||||
.to_string_lossy()
|
||||
.replace('\\', "/");
|
||||
let raw = std::fs::read_to_string(&path)
|
||||
.map_err(|e| anyhow::anyhow!("reading {}: {e}", path.display()))?;
|
||||
out.push((rel, raw));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The offline counterpart to `check`'s needs.yaml fetch -
|
||||
/// `needs.yaml` sits at the repo root beside `aggregates.yaml`.
|
||||
pub fn load_needs_from_dir(dir: &str) -> anyhow::Result<Option<String>> {
|
||||
let needs_path = std::path::Path::new(dir)
|
||||
.parent()
|
||||
.unwrap_or_else(|| std::path::Path::new("."))
|
||||
.join("needs.yaml");
|
||||
if !needs_path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
std::fs::read_to_string(&needs_path)
|
||||
.map(Some)
|
||||
.map_err(|e| anyhow::anyhow!("reading {}: {e}", needs_path.display()))
|
||||
}
|
||||
|
||||
Vendored
+54
@@ -0,0 +1,54 @@
|
||||
//! iris - the shield over a portal site. A content push is an incoming
|
||||
//! wormhole; nothing comes through until it checks out.
|
||||
//!
|
||||
//! `iris check` loads a content repo (a Gitea URL or a local checkout)
|
||||
//! exactly the way the running site does, validates it, and holds it to
|
||||
//! the business needs it declares in `needs.yaml` - a path within
|
||||
//! budget for every kind of visitor, a desk for every handling group,
|
||||
//! every finished state reachable, no record stranding. It also exports
|
||||
//! what a trainer needs to grade the wording: personas as typed choice
|
||||
//! tasks, and a resolved simulation model.
|
||||
//!
|
||||
//! `iris replay` runs simulated cases through the site's real aggregate
|
||||
//! engine on a JetStream and reports every bucket by state; with
|
||||
//! `--report-only` it takes the same scorecard from a live site's
|
||||
//! buckets.
|
||||
//!
|
||||
//! Every type iris reads is portal's own, pinned to the portal release
|
||||
//! it matches, so the lint and the site never disagree about what a
|
||||
//! page is.
|
||||
|
||||
mod check;
|
||||
mod local;
|
||||
mod needs;
|
||||
mod replay;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let mut argv: Vec<String> = std::env::args().skip(1).collect();
|
||||
// `iris --path questions` reads as `iris check --path questions`,
|
||||
// so the content repos' one-line CI keeps working as it was.
|
||||
match argv.first().map(String::as_str) {
|
||||
Some("check") => {
|
||||
argv.remove(0);
|
||||
check::run(argv).await
|
||||
}
|
||||
Some("replay") => {
|
||||
argv.remove(0);
|
||||
replay::run(argv).await
|
||||
}
|
||||
Some("--version") | Some("-V") => {
|
||||
println!("iris {v} (portal v{v})", v = env!("CARGO_PKG_VERSION"));
|
||||
Ok(())
|
||||
}
|
||||
Some(flag) if flag.starts_with("--") => check::run(argv).await,
|
||||
_ => {
|
||||
eprintln!(
|
||||
"iris {v} - matches portal v{v}\n\nusage:\n iris check (--repo <gitea-url> [--branch main] [--subdir questions] | --path <dir>) [--needs-tasks <out.jsonl> [--skim]] [--needs-sim <model.json>] [--access] [--needs-score <answers.jsonl> [--min-accuracy 0.8]]\n iris replay --path <dir> --nats <url> (--cases <cases.jsonl> | --report-only)",
|
||||
v = env!("CARGO_PKG_VERSION")
|
||||
);
|
||||
std::process::exit(2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+1036
File diff suppressed because it is too large
Load Diff
Vendored
+105
@@ -0,0 +1,105 @@
|
||||
//! Makes a simulation's cases real: every case becomes an aggregate in
|
||||
//! its bucket, created and moved by the same code the running site
|
||||
//! uses - `answers::store_answer` for the submission,
|
||||
//! `aggregates::transition` for each desk move, against a real
|
||||
//! JetStream with its event log and its refusal of undeclared moves.
|
||||
//! Then the buckets are read back by replaying each record's events,
|
||||
//! and reported per need against `done_when` and `success`.
|
||||
//!
|
||||
//! The report half runs on its own (`--report-only`) against any
|
||||
//! JetStream, which is how the same scorecard is taken from a live
|
||||
//! site's buckets.
|
||||
//!
|
||||
//! Point it at a throwaway NATS, never a site's own: it writes.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
use futures::StreamExt;
|
||||
use portal::events::store::{ensure_stream, load_events};
|
||||
use crate::local::{load_aggregates_from_dir, load_from_dir, load_needs_from_dir};
|
||||
use crate::needs;
|
||||
use portal::{aggregates, answers, content};
|
||||
|
||||
pub async fn run(argv: Vec<String>) -> anyhow::Result<()> {
|
||||
let value = |flag: &str| argv.iter().position(|a| a == flag).and_then(|i| argv.get(i + 1)).cloned();
|
||||
let usage = "usage: iris replay --path <questions dir> --nats <url> (--cases <cases.jsonl> | --report-only)";
|
||||
let dir = value("--path").ok_or_else(|| anyhow::anyhow!(usage))?;
|
||||
let nats_url = value("--nats").ok_or_else(|| anyhow::anyhow!(usage))?;
|
||||
let cases_path = value("--cases");
|
||||
if cases_path.is_none() && !argv.iter().any(|a| a == "--report-only") {
|
||||
anyhow::bail!(usage);
|
||||
}
|
||||
|
||||
let questions = load_from_dir(&dir)?;
|
||||
let schemas = content::with_builtin_aggregates(load_aggregates_from_dir(&dir)?);
|
||||
let file = needs::parse(&load_needs_from_dir(&dir)?.ok_or_else(|| anyhow::anyhow!("the repo has no needs.yaml"))?)?;
|
||||
let js = async_nats::jetstream::new(async_nats::connect(&nats_url).await?);
|
||||
// The same call the site makes at boot.
|
||||
ensure_stream(&js).await?;
|
||||
|
||||
if let Some(path) = cases_path {
|
||||
let raw = std::fs::read_to_string(&path).map_err(|e| anyhow::anyhow!("reading {path}: {e}"))?;
|
||||
let (mut created, mut moved, mut refused) = (0, 0, 0);
|
||||
for (n, line) in raw.lines().filter(|l| !l.trim().is_empty()).enumerate() {
|
||||
let case: serde_json::Value = serde_json::from_str(line)?;
|
||||
let Some(need) = file.needs.iter().find(|need| case["need"] == need.id.as_str()) else { continue };
|
||||
let Some(schema) = schemas.get(&need.lands_in) else { continue };
|
||||
// Only a case that got as far as submitting leaves a record.
|
||||
let states: Vec<&str> = case["states"].as_array().into_iter().flatten().filter_map(|s| s.as_str()).collect();
|
||||
if states.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let step = needs::path_for(&questions, need).and_then(|p| p.last().cloned());
|
||||
let (page, alternative) = step.map(|s| (s.page, s.alternative)).unwrap_or_default();
|
||||
let id = format!("sim-{}-{}-{n}", case["persona"].as_str().unwrap_or("x"), case["run"]);
|
||||
let now = chrono::Utc::now().timestamp_millis();
|
||||
answers::store_answer(&js, &schemas, &need.lands_in, id.clone(), &page, &alternative, &case["record"], now).await?;
|
||||
created += 1;
|
||||
for target in &states[1..] {
|
||||
match aggregates::transition(&js, schema, &id, target, serde_json::json!({"by": "simulation"}), now).await {
|
||||
Ok(_) => moved += 1,
|
||||
Err(e) => {
|
||||
refused += 1;
|
||||
eprintln!("REFUSED by the engine: {id} -> {target}: {e:?}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("{created} aggregate(s) created, {moved} transition(s) accepted, {refused} refused by the engine\n");
|
||||
}
|
||||
|
||||
// Read every bucket back the way the engine knows it: by replaying
|
||||
// each record's own event log.
|
||||
let mut by_bucket: HashMap<String, BTreeMap<String, usize>> = HashMap::new();
|
||||
for bucket in file.needs.iter().map(|n| n.lands_in.clone()).collect::<std::collections::BTreeSet<_>>() {
|
||||
let (Some(schema), Ok(store)) = (schemas.get(&bucket), js.get_key_value(&bucket).await) else { continue };
|
||||
let mut keys = store.keys().await?;
|
||||
while let Some(Ok(id)) = keys.next().await {
|
||||
let events = load_events(&js, &bucket, &id).await?;
|
||||
if let Some(record) = aggregates::replay(schema, &id, &events) {
|
||||
*by_bucket.entry(bucket.clone()).or_default().entry(record.state).or_default() += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("{:<18} {:>8} {:>9} {:>9} {}", "bucket", "records", "finished", "success", "by state");
|
||||
let mut buckets: Vec<_> = by_bucket.iter().collect();
|
||||
buckets.sort();
|
||||
for (bucket, states) in buckets {
|
||||
let needs_here: Vec<&needs::Need> = file.needs.iter().filter(|n| &n.lands_in == bucket).collect();
|
||||
let is = |pick: fn(&needs::Need) -> &Vec<String>, state: &String| needs_here.iter().any(|n| pick(n).contains(state));
|
||||
let total: usize = states.values().sum();
|
||||
let finished: usize = states.iter().filter(|(s, _)| is(|n| &n.done_when, s)).map(|(_, c)| c).sum();
|
||||
let success: usize = states.iter().filter(|(s, _)| is(|n| &n.success, s)).map(|(_, c)| c).sum();
|
||||
let pct = |n: usize| if total == 0 { "-".to_string() } else { format!("{:.0}%", n as f64 * 100.0 / total as f64) };
|
||||
println!(
|
||||
"{:<18} {:>8} {:>9} {:>9} {}",
|
||||
bucket,
|
||||
total,
|
||||
pct(finished),
|
||||
pct(success),
|
||||
states.iter().map(|(s, c)| format!("{s} {c}")).collect::<Vec<_>>().join(", ")
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user