From 7caaa27ba7e604641f7ca2ab5767d31f2cbc90a4 Mon Sep 17 00:00:00 2001 From: Bendik Aagaard Lynghaug Date: Mon, 21 Sep 2026 16:31:37 +0200 Subject: [PATCH 001/110] portal-trainer: the loop around portal's needs verifier The trainer procedure as an agent skill, the engine contract, two sector specs (software-studio: UHHM's own, green; construction: a target with no pages yet), and the first wording run against uhhm.no. The verifier itself is portal's question_lint (uhhm/portal 188f9bf). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01L4jrCgLiKKHAEFuZUJjckH --- README.md | 123 ++++++++++++++++++ runs/uhhm-2026-09-21/answers-full.jsonl | 8 ++ runs/uhhm-2026-09-21/answers-skim.jsonl | 8 ++ runs/uhhm-2026-09-21/tasks.jsonl | 8 ++ sectors/construction/needs.yaml | 158 ++++++++++++++++++++++++ sectors/software-studio/needs.yaml | 127 +++++++++++++++++++ skill/SKILL.md | 136 ++++++++++++++++++++ 7 files changed, 568 insertions(+) create mode 100644 README.md create mode 100644 runs/uhhm-2026-09-21/answers-full.jsonl create mode 100644 runs/uhhm-2026-09-21/answers-skim.jsonl create mode 100644 runs/uhhm-2026-09-21/tasks.jsonl create mode 100644 sectors/construction/needs.yaml create mode 100644 sectors/software-studio/needs.yaml create mode 100644 skill/SKILL.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..f022528 --- /dev/null +++ b/README.md @@ -0,0 +1,123 @@ +# portal-trainer + +Onboarding a business onto [portal](https://project.uhhm.no/uhhm/portal) +is mostly a wording problem: the engine is generic, and whether it +works for a given business depends on whether its pages say, in the +visitor's words, the thing the business needs them to answer. This +repo is where that gets tested and, eventually, learned. + +## The loop + +``` +interview ─► needs.yaml ─► pages + aggregates + desks + │ │ + │ question_lint (structure: pass/fail) + │ │ + └─► personas ─► tasks.jsonl ─► answering engine ─► answers.jsonl + │ + question_lint --needs-score ◄──┘ + │ + misroutes = wording defects ─► revise pages +``` + +- **`needs.yaml`** lives in the business's own content repo. It says + who wants what, which bucket the answer lands in, which group + carries it to a finished state, and what the visitor will tolerate + in submissions and required fields. +- **The verifier is portal's `question_lint`**, not code in this repo. + It reads portal's own content types and ships in every portal + release, so a content repo's CI enforces its needs with no new + dependency. Structure is deterministic and gates CI. Wording is + graded, and gates only if you pass `--min-accuracy`. +- **This repo holds what sits around the verifier**: the trainer + procedure (`skill/SKILL.md`), sector specs to start from + (`sectors/`), and the runs (`runs/`). + +``` +question_lint --path /questions # structure +question_lint --path /questions --needs-tasks tasks.jsonl # full wording +question_lint --path /questions --needs-tasks skim.jsonl --skim # headings only +question_lint --path /questions --needs-score answers.jsonl [--min-accuracy 0.8] +``` + +## The engine contract + +A task is one page on one persona's path, as a typed `choice`: + +```json +{"id":"architecture-firm@/","persona":"architecture-firm","page":"/", + "state":{"body":"We run a small architecture firm and ..."}, + "questions":{"pick":{"type":"choice", + "instructions":"A visitor reads the page question \"What are you here for?\". Which alternative do they choose?", + "criteria":{"":"", "...":"..."}}}, + "expect":[""]} +``` + +Strip `expect` (and `need`) before an engine sees it. An answer is one +line: `{"id": "...", "choice": "", "confidence": 0.0-1.0}`. +`state` and `questions` are exactly the arguments of +[laya](https://github.com/NandhaKishorM/laya)'s +`router.predict(state, questions)`, so a laya adapter is a loop over +the file; an LLM adapter is a prompt around the same two fields. + +## Engines, and what is known about them + +| Engine | Status | What it is good for | +|---|---|---| +| LLM, blind (no `expect`) | used for the first run | Reads like a visitor; slow and costs per task, so use it to find defects, not to gate CI. | +| laya `english` (ModernBERT-large, 421M) | not run yet | 33 ms per question, so it can gate CI and score thousands of generated variants. Its authors measure 0.78 intent accuracy in English. | +| laya `multilingual` | not run yet | Its authors' own sweep puts Norwegian Bokmål at **0.53 accuracy with 0.86 mean confidence**. It is confidently wrong too often to grade Norwegian wording without fine-tuning. Test a Norwegian site in English translation, or with an LLM. | + +An engine is a measuring instrument. Before trusting one on a sector, +score it on tasks whose right answer nobody disputes; an engine that +misses those is measuring itself, not the page. + +## First run: uhhm.no, 2026-09-21 + +Nine personas against the live uhhm.no front door, answered by a small +LLM that never saw `expect`. One run each; eight tasks; treat as a +lead, not a measurement. + +| What the engine could read | Routed right | +|---|---| +| Headings and descriptions | 7 of 8 | +| Headings only (`--skim`) | 5 of 8 | + +- A founder who wants a first version built picked **"Work happens + through dialogue, not tickets"** - the collaborator door - in both + modes. With headings only, so did a firm wanting to pay for storage + and backups. The heading reads as how UHHM works with clients. +- With headings only, a student seeking an internship picked + **"Attention is earned"**. +- Structure: four needs met. The site-improvement flow is advertised + and `disabled`, so it is declared `planned`. + +## Sectors + +A sector file is a needs spec with no pages behind it: a failing test +suite for the trainer to make pass, carrying what the sector demands +(why a lead is a pipeline, why a deviation report gets three fields). + +- `sectors/software-studio/needs.yaml` - UHHM's own, and green against + `uhhm/questions`. +- `sectors/construction/needs.yaml` - a contractor with project + managers, a safety officer, crew and subcontractors: winning survey + and design work, vetting subcontractors, requesting and answering + bids, hiring, deviation reporting. No pages yet. + +## Toward a trained model + +Nothing here trains weights yet, on purpose. The order that makes +training possible: + +1. The verifier has to be trusted first. It is the reward. A generator + trained against a judge that is wrong half the time learns the + judge. +2. Every onboarding leaves a run: needs, the pages at each revision, + the scores. That corpus - what wording routed which persona right, + per sector - is the training data, and it does not exist until the + loop has been run by hand a few dozen times. +3. With a fast engine (laya-class) as the scorer, generating and + scoring many page variants per need is cheap, which is what makes + search, and later fine-tuning a small generator on the winners, + practical. diff --git a/runs/uhhm-2026-09-21/answers-full.jsonl b/runs/uhhm-2026-09-21/answers-full.jsonl new file mode 100644 index 0000000..eb8f7c1 --- /dev/null +++ b/runs/uhhm-2026-09-21/answers-full.jsonl @@ -0,0 +1,8 @@ +{"id":"architecture-firm@/","choice":"Good software begins with a clear argument","confidence":0.9} +{"id":"climbing-gym-founder@/","choice":"Work happens through dialogue, not tickets","confidence":0.75} +{"id":"municipality-buyer@/","choice":"Good software begins with a clear argument","confidence":0.95} +{"id":"exhibition-visitor@/","choice":"Attention is earned","confidence":0.95} +{"id":"tech-journalist@/","choice":"Attention is earned","confidence":0.9} +{"id":"rust-developer@/","choice":"Work happens through dialogue, not tickets","confidence":0.8} +{"id":"interaction-designer@/","choice":"Work happens through dialogue, not tickets","confidence":0.85} +{"id":"student-intern@/","choice":"Work happens through dialogue, not tickets","confidence":0.65} diff --git a/runs/uhhm-2026-09-21/answers-skim.jsonl b/runs/uhhm-2026-09-21/answers-skim.jsonl new file mode 100644 index 0000000..58fa88f --- /dev/null +++ b/runs/uhhm-2026-09-21/answers-skim.jsonl @@ -0,0 +1,8 @@ +{"id":"architecture-firm@/","choice":"Work happens through dialogue, not tickets","confidence":0.70} +{"id":"climbing-gym-founder@/","choice":"Work happens through dialogue, not tickets","confidence":0.85} +{"id":"municipality-buyer@/","choice":"Good software begins with a clear argument","confidence":0.60} +{"id":"exhibition-visitor@/","choice":"Attention is earned","confidence":0.80} +{"id":"tech-journalist@/","choice":"Attention is earned","confidence":0.80} +{"id":"rust-developer@/","choice":"Work happens through dialogue, not tickets","confidence":0.75} +{"id":"interaction-designer@/","choice":"Work happens through dialogue, not tickets","confidence":0.80} +{"id":"student-intern@/","choice":"Attention is earned","confidence":0.55} diff --git a/runs/uhhm-2026-09-21/tasks.jsonl b/runs/uhhm-2026-09-21/tasks.jsonl new file mode 100644 index 0000000..0a0bb8b --- /dev/null +++ b/runs/uhhm-2026-09-21/tasks.jsonl @@ -0,0 +1,8 @@ +{"id":"architecture-firm@/","persona":"architecture-firm","need":"commission","page":"/","state":{"body":"We run a small architecture firm and our project files are scattered across three cloud drives. We want to pay someone to set up proper storage and backups we control ourselves."},"questions":{"pick":{"type":"choice","instructions":"A visitor reads the page question \"What are you here for?\". Which alternative do they choose?","criteria":{"Attention is earned":"Considered notes on what we're building, and why — sent only when there's something worth saying.","Good software begins with a clear argument":"Infrastructure, tools, or a product — bring the problem, and we read the whole brief before answering.","Work happens through dialogue, not tickets":"What you bring shapes what gets built. We onboard collaborators the same deliberate way we build — by hand, through a real conversation."}}},"expect":["Good software begins with a clear argument"]} +{"id":"climbing-gym-founder@/","persona":"climbing-gym-founder","need":"commission","page":"/","state":{"body":"I have an idea for a booking app for climbing gyms and I'm looking for a team that can build the first version with me."},"questions":{"pick":{"type":"choice","instructions":"A visitor reads the page question \"What are you here for?\". Which alternative do they choose?","criteria":{"Attention is earned":"Considered notes on what we're building, and why — sent only when there's something worth saying.","Good software begins with a clear argument":"Infrastructure, tools, or a product — bring the problem, and we read the whole brief before answering.","Work happens through dialogue, not tickets":"What you bring shapes what gets built. We onboard collaborators the same deliberate way we build — by hand, through a real conversation."}}},"expect":["Good software begins with a clear argument"]} +{"id":"municipality-buyer@/","persona":"municipality-buyer","need":"commission","page":"/","state":{"body":"Our municipality needs an internal tool for tracking building permits. We are looking for a vendor who can scope it and deliver it."},"questions":{"pick":{"type":"choice","instructions":"A visitor reads the page question \"What are you here for?\". Which alternative do they choose?","criteria":{"Attention is earned":"Considered notes on what we're building, and why — sent only when there's something worth saying.","Good software begins with a clear argument":"Infrastructure, tools, or a product — bring the problem, and we read the whole brief before answering.","Work happens through dialogue, not tickets":"What you bring shapes what gets built. We onboard collaborators the same deliberate way we build — by hand, through a real conversation."}}},"expect":["Good software begins with a clear argument"]} +{"id":"exhibition-visitor@/","persona":"exhibition-visitor","need":"follow","page":"/","state":{"body":"I saw your drawing network at an exhibition and liked it. I don't want to get involved, I'd just like to know when there is news."},"questions":{"pick":{"type":"choice","instructions":"A visitor reads the page question \"What are you here for?\". Which alternative do they choose?","criteria":{"Attention is earned":"Considered notes on what we're building, and why — sent only when there's something worth saying.","Good software begins with a clear argument":"Infrastructure, tools, or a product — bring the problem, and we read the whole brief before answering.","Work happens through dialogue, not tickets":"What you bring shapes what gets built. We onboard collaborators the same deliberate way we build — by hand, through a real conversation."}}},"expect":["Attention is earned"]} +{"id":"tech-journalist@/","persona":"tech-journalist","need":"follow","page":"/","state":{"body":"I write about Nordic technology. Keep me posted on what you release so I can cover it."},"questions":{"pick":{"type":"choice","instructions":"A visitor reads the page question \"What are you here for?\". Which alternative do they choose?","criteria":{"Attention is earned":"Considered notes on what we're building, and why — sent only when there's something worth saying.","Good software begins with a clear argument":"Infrastructure, tools, or a product — bring the problem, and we read the whole brief before answering.","Work happens through dialogue, not tickets":"What you bring shapes what gets built. We onboard collaborators the same deliberate way we build — by hand, through a real conversation."}}},"expect":["Attention is earned"]} +{"id":"rust-developer@/","persona":"rust-developer","need":"collaborate","page":"/","state":{"body":"I'm a Rust developer between jobs and I like that you build in the open. I would love to contribute, and maybe join you."},"questions":{"pick":{"type":"choice","instructions":"A visitor reads the page question \"What are you here for?\". Which alternative do they choose?","criteria":{"Attention is earned":"Considered notes on what we're building, and why — sent only when there's something worth saying.","Good software begins with a clear argument":"Infrastructure, tools, or a product — bring the problem, and we read the whole brief before answering.","Work happens through dialogue, not tickets":"What you bring shapes what gets built. We onboard collaborators the same deliberate way we build — by hand, through a real conversation."}}},"expect":["Work happens through dialogue, not tickets"]} +{"id":"interaction-designer@/","persona":"interaction-designer","need":"collaborate","page":"/","state":{"body":"I'm an interaction designer. I want to work with you on the drawing interface."},"questions":{"pick":{"type":"choice","instructions":"A visitor reads the page question \"What are you here for?\". Which alternative do they choose?","criteria":{"Attention is earned":"Considered notes on what we're building, and why — sent only when there's something worth saying.","Good software begins with a clear argument":"Infrastructure, tools, or a product — bring the problem, and we read the whole brief before answering.","Work happens through dialogue, not tickets":"What you bring shapes what gets built. We onboard collaborators the same deliberate way we build — by hand, through a real conversation."}}},"expect":["Work happens through dialogue, not tickets"]} +{"id":"student-intern@/","persona":"student-intern","need":"collaborate","page":"/","state":{"body":"I'm a computer science student looking for a summer internship doing infrastructure work."},"questions":{"pick":{"type":"choice","instructions":"A visitor reads the page question \"What are you here for?\". Which alternative do they choose?","criteria":{"Attention is earned":"Considered notes on what we're building, and why — sent only when there's something worth saying.","Good software begins with a clear argument":"Infrastructure, tools, or a product — bring the problem, and we read the whole brief before answering.","Work happens through dialogue, not tickets":"What you bring shapes what gets built. We onboard collaborators the same deliberate way we build — by hand, through a real conversation."}}},"expect":["Work happens through dialogue, not tickets"]} diff --git a/sectors/construction/needs.yaml b/sectors/construction/needs.yaml new file mode 100644 index 0000000..b0754b2 --- /dev/null +++ b/sectors/construction/needs.yaml @@ -0,0 +1,158 @@ +# A building contractor's needs, as a portal needs spec: a small to +# mid-sized firm doing surveys, design and rehabilitation, run by a few +# project managers with its own crew and a ring of subcontractors. +# +# This file is a target, not a description: there are no pages behind +# it yet, so `question_lint` fails every need. The trainer's job is to +# write the pages, aggregates.yaml and desks that turn it green, then +# word them until the personas route right. +# +# Groups (Kanidm): project_managers, hse, crew, subcontractors. +# Everyone else arrives anonymous. +# +# What the sector demands of the flows, and why the budgets are what +# they are: +# - Winning work is a pipeline, not an inbox: a lead is surveyed, then +# quoted, then won or lost. "answered" is not an outcome. +# - A subcontractor must be vetted before they set foot on site +# (company registration, trade approval, insurance, tax standing, +# site ID cards for their people). Documents go missing; the state +# machine has to say so and let them come back. +# - Deviation and near-miss reporting only works when it costs the +# reporter seconds. Three required fields, one submission, from a +# phone, and the safety officer owns closing it. +# - People move between firms constantly; onboarding a worker is the +# same shape as onboarding a subcontractor, one size smaller. +needs: + - id: win-survey-work + who: a building owner or housing board with a problem in their building + wants: someone qualified to look at it and say what it takes + lands_in: leads + handled_by: project_managers + done_when: [won, lost] + max_steps: 2 + max_fields: 5 + + - id: win-design-work + who: an owner who knows what they want built or changed + wants: drawings, a permit application, and a price + lands_in: leads + handled_by: project_managers + done_when: [won, lost] + max_steps: 2 + max_fields: 6 + + - id: vet-subcontractor + who: a trade firm that wants to work for us + wants: to be approved once, and then be asked + lands_in: subcontractors + handled_by: project_managers + done_when: [approved, rejected] + max_steps: 2 + max_fields: 7 + + - id: request-bids + who: a project manager with a job to staff + wants: approved subcontractors to see it and answer + enters: /jobs/post + enters_as: project_managers + lands_in: job_requests + handled_by: project_managers + done_when: [awarded, withdrawn] + max_steps: 1 + max_fields: 5 + + - id: answer-bid + who: an approved subcontractor + wants: to say yes to a job, with a price and a start date + enters: /jobs + enters_as: subcontractors + lands_in: bids + handled_by: project_managers + done_when: [accepted, declined] + max_steps: 2 + max_fields: 3 + + - id: hire-crew + who: a tradesperson looking for work + wants: to be considered, without writing a letter + lands_in: applicants + handled_by: project_managers + done_when: [hired, declined] + max_steps: 2 + max_fields: 4 + + - id: report-deviation + who: anyone on site - crew or a subcontractor's people + wants: to report a hazard, a near miss or a defect before they forget + enters: /site + enters_as: crew + lands_in: deviations + handled_by: hse + done_when: [closed] + max_steps: 1 + max_fields: 3 + +personas: + - id: housing-board-damp + need: win-survey-work + text: > + I chair the board of a housing cooperative from the seventies. + Two ground-floor flats have damp in the walls and we don't know + if it's drainage or the facade. We need someone to tell us. + + - id: buyer-before-purchase + need: win-survey-work + text: > + We are about to buy an old timber house. Before we sign we want + a professional to go through it and tell us what is wrong with it. + + - id: attic-conversion + need: win-design-work + text: > + We want to turn our attic into two bedrooms and a bathroom. We + need drawings and someone who handles the application to the + municipality. + + - id: shop-to-flat + need: win-design-work + text: > + I own a former shop on the ground floor and want to make it a + flat. I have no idea what is allowed. Can you sort that out and + give me a price? + + - id: electrician-firm + need: vet-subcontractor + text: > + We are a four-person electrical firm, fully certified. We would + like to get on your list so you call us when you need + electricians. + + - id: scaffolder + need: vet-subcontractor + text: > + I run a scaffolding company. How do I get to do jobs for you? + + - id: carpenter-jobseeker + need: hire-crew + text: > + I'm a certified carpenter with eight years on rehabilitation + jobs. Are you hiring? + + - id: apprentice + need: hire-crew + text: > + I finish vocational school this spring and need an + apprenticeship as a bricklayer. + + - id: loose-railing + need: report-deviation + text: > + The railing on the third level of the scaffold on the north + side is loose. Nobody got hurt but someone will. + + - id: wrong-membrane + need: report-deviation + text: > + The bathroom membrane delivered today is not the one in the + specification. We stopped work on it. diff --git a/sectors/software-studio/needs.yaml b/sectors/software-studio/needs.yaml new file mode 100644 index 0000000..a8ab2fe --- /dev/null +++ b/sectors/software-studio/needs.yaml @@ -0,0 +1,127 @@ +# What UHHM, as a business, needs this site to do - stated apart from +# how the pages word it. `question_lint` checks every need against the +# pages and aggregates.yaml (portal `src/needs.rs`): a path of +# submissions gets the visitor's answer into `lands_in` within the +# effort budget, `handled_by` has a desk over that bucket, and every +# `done_when` state is reachable through the moves the desk offers. +# +# The personas are visitors in their own words - never the site's. +# `question_lint --needs-tasks` turns each page on their path into a +# typed choice for an answering engine; `--needs-score` grades its +# picks. A misroute means the page said one thing and they heard +# another. The runtime never reads this file. +needs: + - id: commission + who: someone with a problem software could solve, and a budget + wants: UHHM to take the work on + via: Good software begins with a clear argument + lands_in: projects + handled_by: owners + done_when: [accepted, declined] + max_steps: 1 + # name, email, the build, the organization - the page argues for + # each of the four; a fifth would need the same argument. + max_fields: 4 + + - id: follow + who: someone curious who isn't ready to get involved + wants: to hear when something ships + via: Attention is earned + lands_in: subscribers + handled_by: owners + # Being on the list is the finished state; leaving is theirs to do. + done_when: [open, unsubscribed] + max_steps: 1 + max_fields: 2 + + - id: collaborate + who: a developer, designer or researcher + wants: to work with UHHM + via: Work happens through dialogue, not tickets + lands_in: applicants + handled_by: owners + done_when: [active, declined] + max_steps: 1 + max_fields: 3 + + - id: improve-site + who: anyone using this site + wants: the site itself to change + enters: /develop/proposal + # Announced, not open: the automation that turns a proposal into a + # pull request is still being built, so the alternative is disabled. + planned: true + lands_in: development_proposals + handled_by: owners + done_when: [approved, rejected] + max_steps: 1 + max_fields: 3 + + - id: track-prospect + who: an owner who met an organization worth following up + wants: the relationship written down and walked to an outcome + enters: /review + enters_as: owners + via: Record a prospect + lands_in: organizations + handled_by: owners + done_when: [client, past_client] + max_steps: 1 + max_fields: 2 + +personas: + - id: architecture-firm + need: commission + text: > + We run a small architecture firm and our project files are + scattered across three cloud drives. We want to pay someone to + set up proper storage and backups we control ourselves. + + - id: climbing-gym-founder + need: commission + text: > + I have an idea for a booking app for climbing gyms and I'm + looking for a team that can build the first version with me. + + - id: municipality-buyer + need: commission + text: > + Our municipality needs an internal tool for tracking building + permits. We are looking for a vendor who can scope it and + deliver it. + + - id: exhibition-visitor + need: follow + text: > + I saw your drawing network at an exhibition and liked it. I don't + want to get involved, I'd just like to know when there is news. + + - id: tech-journalist + need: follow + text: > + I write about Nordic technology. Keep me posted on what you + release so I can cover it. + + - id: rust-developer + need: collaborate + text: > + I'm a Rust developer between jobs and I like that you build in + the open. I would love to contribute, and maybe join you. + + - id: interaction-designer + need: collaborate + text: > + I'm an interaction designer. I want to work with you on the + drawing interface. + + - id: student-intern + need: collaborate + text: > + I'm a computer science student looking for a summer internship + doing infrastructure work. + + - id: site-critic + need: improve-site + text: > + Your contact form never says how long a reply takes. You should + add that. diff --git a/skill/SKILL.md b/skill/SKILL.md new file mode 100644 index 0000000..927d871 --- /dev/null +++ b/skill/SKILL.md @@ -0,0 +1,136 @@ +--- +name: portal-trainer +description: Onboard a business onto portal - interview them, write their needs.yaml, then write the content repo (pages, aggregates.yaml, desks) until question_lint proves every need is met and an answering engine routes the personas right. Use when someone wants a new portal site, a new flow on an existing one, or asks why visitors pick the wrong alternative. +--- + +# Portal trainer + +You are onboarding a business onto [portal](https://project.uhhm.no/uhhm/portal): +a site that is one question per page, a few alternatives per question, +each alternative optionally a form whose answer is recorded into a +bucket with a state machine, worked from a group-gated desk. + +You have a verifier. Use it instead of your judgment wherever it can +speak: `question_lint --path /questions` exits non-zero until +the structure meets every need in `/needs.yaml`. + +## The order of work + +1. **Interview, then write `needs.yaml` first.** Never pages first. + Ask who comes to the business and what each of them wants, who in + the business handles it, and what "handled" means - the states a + case ends in. Ask what it costs the visitor to ask: how many + submissions, how many required fields they will tolerate. If the + business is in a sector under `sectors/`, start from that file and + cut it down with them; do not start blank. +2. **Write personas in the visitor's words, not the business's.** Two + or three per need, including one who would not use the trade's own + vocabulary. Never reuse a phrase you intend to put on a page - a + persona that quotes the page tests nothing. +3. **Run the lint. It fails. Read every line.** Each failure names + what is missing: no path, a closed (`disabled`) path, a field budget + blown, no desk held by the handling group, a finished state no desk + move reaches, a state records strand in. +4. **Write the least content that turns it green**: `aggregates.yaml` + (one aggregate per bucket, states the business named, only the + transitions they described), the visitor pages, one followup page + per recording alternative saying what happens next, and one desk + page per handling group with exactly the moves the state machine + allows. Mark a need `planned: true` only when the business says + "later" - it keeps warning. +5. **Then word it.** `question_lint ... --needs-tasks tasks.jsonl`, + and again with `--skim` (headings only - most visitors read nothing + else). Give the tasks to an answering engine that never sees + `expect`; grade with `--needs-score answers.jsonl`. + - A **misroute** is a wording defect. Fix the heading before the + description: a heading that needs its description has failed. + - Two alternatives a persona confuses are usually one distinction + the business makes and the visitor does not. Name the visitor's + side of it. + - **Hesitant** (right, under 0.6) means it works and is still a coin + flip. Same treatment. + - Change the page, never the persona, to make a score go up. +6. **Keep the run.** Copy tasks, answers and the score into + `runs/-/` with a line on which engine answered. + Runs are the record of which wording worked, per sector. + +## What a page is + +Full reference: portal's `README.md` and +`docs/design/filesystem-routes.md`; a live example is +`uhhm/questions`. The file tree under `questions/` is the router +(`index.yaml` is `/`, `_section.yaml` applies `qualifies` to a +directory, nested non-index files are followups). + +```yaml +# questions/index.yaml +name: What brings you here? # the question - always a question +responsible: { name: ..., contact: ... } +alternatives: + - name: # graded alone under --skim + description: + action: /received # where the submission lands them + consequence: [Send] # the button + record_as: leads # bucket - must exist in aggregates.yaml + features: + - name:
+ requirements: + - { name: email, label: Email, type: email } + - { name: problem, label: What's wrong, type: textarea } + - { name: address, label: Address, optional: true } +``` + +```yaml +# aggregates.yaml +aggregates: + - bucket: leads + initial: open + states: + open: { event: received } + quoted: { event: quoted } + won: { event: won } + lost: { event: lost } + transitions: + open: [quoted, lost] + quoted: [won, lost] +``` + +```yaml +# questions/desk/_section.yaml +qualifies: project_managers +# questions/desk/index.yaml +name: What came in? +alternatives: + - name: Leads + action: /desk + consequence: [Confirm] + features: + - name: "" + resource: + source: { kind: kv, bucket: leads } + requires_group: project_managers + transitions: + - { to: quoted, label: Quote sent } # from defaults to open + - { to: lost, label: Not for us } + - { from: quoted, to: won, label: Won } + - { from: quoted, to: lost, label: Lost } +``` + +## House rules for the copy + +- The page name is a question the visitor could ask themselves. +- Headings say what the visitor gets, in the visitor's words. Lead + with what a thing enables; never define it by what it isn't. +- Every required field must be one the business will actually use to + answer. If they can't say what they do with it, it's optional or + gone. +- A followup page says who reads it, when they answer, and what the + visitor can do meanwhile. + +## Engines + +Any engine that reads a task's `state` and `questions` and returns +`{"id","choice","confidence"}` per line works. Compare at least two +before trusting a score, and re-run the same engine before trusting a +change: one run of eight tasks is an anecdote. See the repo README for +what has been measured about each. -- 2.54.0 From b6bdac6f6269224e3f9a455656337c50604d871f Mon Sep 17 00:00:00 2001 From: Bendik Aagaard Lynghaug Date: Mon, 21 Sep 2026 18:14:25 +0200 Subject: [PATCH 002/110] Generator, LLM backends, and the onboarding site `portal-trainer generate` turns a plain-words business intake into a content repo in three phases, each held to portal's question_lint: needs.yaml (then frozen, so the generator cannot weaken the spec to pass), structure (revised against the lint's findings until green), and wording (personas answered blind, misroutes fed back; a rewrite that breaks structure is reverted). Model output is written to disk only under aggregates.yaml, site.yaml and questions/**.yaml. `portal-trainer answer` is the blind answerer alone. The LLM is an external command, prompt on stdin, text on stdout: engines/ollama.sh (local Ollama), engines/anthropic.sh (API key), engines/mailbox.sh (hand each prompt to an agent or a person). portal-portal/ is the onboarding site as a portal content repo: the front door is the intake interview, records land in `onboardings` (open -> drafted <-> revising -> live | declined), and it passes its own needs spec. runs/uhhm-2026-09-21/ollama: qwen3:8b and granite4.1:8b as blind answerers on the same eight tasks. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01L4jrCgLiKKHAEFuZUJjckH --- .gitignore | 2 + Cargo.lock | 162 ++++++++ Cargo.toml | 11 + engines/anthropic.sh | 11 + engines/mailbox.sh | 12 + engines/ollama.sh | 13 + generator/reference.md | 179 ++++++++ intakes/contractor.yaml | 44 ++ portal-portal/aggregates.yaml | 17 + portal-portal/needs.yaml | 41 ++ portal-portal/questions/desk/_section.yaml | 1 + portal-portal/questions/desk/index.yaml | 25 ++ portal-portal/questions/how.yaml | 21 + portal-portal/questions/index.yaml | 49 +++ portal-portal/questions/received.yaml | 15 + portal-portal/site.yaml | 2 + .../ollama/ans-full-granite4.1.jsonl | 8 + .../ollama/ans-full-qwen3.jsonl | 8 + .../ollama/ans-skim-granite4.1.jsonl | 8 + .../ollama/ans-skim-qwen3.jsonl | 8 + src/main.rs | 392 ++++++++++++++++++ 21 files changed, 1029 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100755 engines/anthropic.sh create mode 100755 engines/mailbox.sh create mode 100755 engines/ollama.sh create mode 100644 generator/reference.md create mode 100644 intakes/contractor.yaml create mode 100644 portal-portal/aggregates.yaml create mode 100644 portal-portal/needs.yaml create mode 100644 portal-portal/questions/desk/_section.yaml create mode 100644 portal-portal/questions/desk/index.yaml create mode 100644 portal-portal/questions/how.yaml create mode 100644 portal-portal/questions/index.yaml create mode 100644 portal-portal/questions/received.yaml create mode 100644 portal-portal/site.yaml create mode 100644 runs/uhhm-2026-09-21/ollama/ans-full-granite4.1.jsonl create mode 100644 runs/uhhm-2026-09-21/ollama/ans-full-qwen3.jsonl create mode 100644 runs/uhhm-2026-09-21/ollama/ans-skim-granite4.1.jsonl create mode 100644 runs/uhhm-2026-09-21/ollama/ans-skim-qwen3.jsonl create mode 100644 src/main.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..90de6fc --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +target/ +out/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..d96719b --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,162 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "portal-trainer" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "serde_yaml", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..7da1273 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "portal-trainer" +version = "0.1.0" +edition = "2021" +description = "Generates a portal content repo from a business intake, held to portal's question_lint" + +[dependencies] +anyhow = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +serde_yaml = "0.9" diff --git a/engines/anthropic.sh b/engines/anthropic.sh new file mode 100755 index 0000000..76b86ea --- /dev/null +++ b/engines/anthropic.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# LLM backend for portal-trainer: prompt on stdin, text on stdout. +# Needs ANTHROPIC_API_KEY, curl and jq. MODEL and MAX_TOKENS override. +set -eu +: "${ANTHROPIC_API_KEY:?set ANTHROPIC_API_KEY}" +jq -Rs --arg model "${MODEL:-claude-sonnet-5}" --argjson max "${MAX_TOKENS:-16000}" \ + '{model: $model, max_tokens: $max, messages: [{role: "user", content: .}]}' | +curl -sS --fail-with-body --max-time "${TIMEOUT:-600}" https://api.anthropic.com/v1/messages \ + -H "x-api-key: $ANTHROPIC_API_KEY" -H "anthropic-version: 2023-06-01" \ + -H "content-type: application/json" --data-binary @- | +jq -er '[.content[] | select(.type == "text") | .text] | join("")' diff --git a/engines/mailbox.sh b/engines/mailbox.sh new file mode 100755 index 0000000..d8277e4 --- /dev/null +++ b/engines/mailbox.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# LLM backend that hands each prompt to whoever watches $MAILBOX: +# prompt-N.md goes out, the reply is read from reply-N.md. For driving +# the generator from an agent session, or by hand. +set -eu +: "${MAILBOX:?set MAILBOX to a directory}" +mkdir -p "$MAILBOX" +n=$(( $(ls "$MAILBOX" | grep -c '^prompt-' || true) + 1 )) +cat > "$MAILBOX/prompt-$n.md" +until [ -s "$MAILBOX/reply-$n.md" ]; do sleep 2; done +sleep 1 +cat "$MAILBOX/reply-$n.md" diff --git a/engines/ollama.sh b/engines/ollama.sh new file mode 100755 index 0000000..707d60c --- /dev/null +++ b/engines/ollama.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# LLM backend for portal-trainer on a local Ollama: prompt on stdin, +# text on stdout. MODEL, OLLAMA_HOST, NUM_CTX and TIMEOUT override. +# Thinking is off: the generator wants files, the answerer wants JSONL. +set -eu +jq -Rs --arg model "${MODEL:-qwen3:8b}" --argjson ctx "${NUM_CTX:-32768}" \ + '{model: $model, stream: false, think: false, + options: {num_ctx: $ctx, temperature: 0.2}, + messages: [{role: "user", content: .}]}' | +curl -sS --fail-with-body --max-time "${TIMEOUT:-1800}" \ + "${OLLAMA_HOST:-http://127.0.0.1:11434}/api/chat" \ + -H "content-type: application/json" --data-binary @- | +jq -er '.message.content' diff --git a/generator/reference.md b/generator/reference.md new file mode 100644 index 0000000..67554ff --- /dev/null +++ b/generator/reference.md @@ -0,0 +1,179 @@ +# Portal content reference (for the generator) + +Portal serves a site that is one question per page. Each page offers a +few alternatives. An alternative may carry a form; submitting it +records the answer into a *bucket* whose records move through a state +machine, worked from a *desk*: a page gated to a group, listing the +bucket with one button per allowed move. There is no other mechanism. +Everything a business needs is expressed as: who answers which +question, where the answer lands, who moves it, through which states. + +A content repo is these files and nothing else: + +``` +needs.yaml what the business needs (the spec you are held to) +aggregates.yaml one state machine per bucket +site.yaml title and language +questions/index.yaml the front door, "/" +questions/.yaml "/" +questions//index.yaml "/" +questions//_section.yaml applies `qualifies` to the whole directory +questions//.yaml "//" - a followup page by default +``` + +## needs.yaml + +```yaml +needs: + - id: kebab-case + who: the kind of person, in plain words + wants: what they want from the business + enters: / # page they arrive on; default / + enters_as: # only when they are logged-in members of a group; omit for the public + via: # only when several alternatives on the entry page record into the same bucket + lands_in: + handled_by: # the role inside the business that must see and move the record + done_when: [, ] # every state that counts as finished, good or bad + max_steps: 1 # submissions before the record exists (1 or 2) + max_fields: 4 # required fields the visitor will tolerate across the path +personas: + - id: kebab-case + need: + text: > + Two or three sentences in the visitor's own voice and vocabulary. +``` + +Rules: +- One need per (kind of person, thing they want). Internal work counts: + a role inside the business recording or posting something is a need + with `enters_as`. +- States come from the business's own description of how a thing + progresses. A pipeline is not an inbox: "answered" is rarely the real + outcome. +- `max_fields` is small on purpose. Reporting something from a phone + gets 3. A first enquiry gets 4 to 6. +- Two or three personas per public need. At least one must avoid the + trade's own vocabulary. A persona must never quote a page heading. + +## aggregates.yaml + +```yaml +aggregates: + - bucket: leads + initial: open + states: # state: { event: past-tense verb } + open: { event: received } + quoted: { event: quoted } + won: { event: won } + lost: { event: lost } + transitions: # only moves the business described + open: [quoted, lost] + quoted: [won, lost] +``` + +The initial state should be `open` (a desk transition with no `from` +means from `open`). + +## A page + +```yaml +name: What brings you here? # always a question the visitor could ask themselves +description: > + One or two sentences. +responsible: { name: , contact: } # required on every entry page +alternatives: + - name: # what the visitor picks. Must work WITHOUT the description. + description: > + One or two sentences. + action: /received # absolute path of an existing page + consequence: [Send] # the button label + record_as: leads # omit for a pure navigation alternative + features: + - name: About the building + description: Optional line of guidance. + requirements: + - { name: email, label: Email, type: email } + - { name: problem, label: What is wrong, type: textarea } + - { name: urgency, label: How soon, type: select, options: [This week, This month, No rush] } + - { name: address, label: Address, optional: true } +``` + +- Field types: `text` (default), `email`, `tel`, `textarea`, `date`, + `select` (must list `options`). Use no other types. +- Every `action` must name a page that exists. After a recording + alternative, send the visitor to a followup page that says who reads + it, when they answer, and what happens next. A followup page is a + top-level file with `followup: true`, or any non-index file in a + subdirectory. Give it one alternative with `action: /` to go back. +- A page for members of a group sets `qualifies: ` (or sits in a + directory whose `_section.yaml` does). Only `enters_as: ` + needs can use it. +- Every bucket that is recorded into must be read by a desk. + +## A desk + +```yaml +# questions/desk/_section.yaml +qualifies: project_managers +``` +```yaml +# questions/desk/index.yaml +name: What came in? +responsible: { name: ..., contact: ... } +alternatives: + - name: Leads + description: One line on what to do with them. + action: /desk + consequence: [Confirm] + features: + - name: "" + description: "" + resource: + source: { kind: kv, bucket: leads } + requires_group: project_managers + transitions: + - { to: quoted, label: Quote sent } + - { to: lost, label: Not for us } + - { from: quoted, to: won, label: Won } + - { from: quoted, to: lost, label: Lost } +``` + +- One desk directory per handling group; one alternative per bucket + that group handles. +- Offer every move the state machine allows and no other. A state with + no move out of it, on the way to a finished state, strands records. +- A group may read a bucket it does not handle: the same `resource` + block with its own `requires_group` and no `transitions`. + +## site.yaml + +```yaml +title: +lang: en +``` + +## Copy rules + +- Headings say what the visitor gets, in the visitor's words. +- Lead with what a thing enables. Never define something by what it + is not. No negations in headings. +- Two alternatives must never be separable only by a distinction the + business makes and the visitor does not. +- Every required field is one the business will use to answer. + +## The verifier + +`question_lint` runs after every attempt. Non-zero exit means you are +not done. Its messages are exact: + +- `no path from ... records into ...`: no chain of alternatives from + the entry page reaches an alternative with that `record_as`. +- `the path asks for N required field(s), over the budget`: mark fields + `optional: true` or drop them. +- `no desk gated on "g" reads "b"`: add the desk. +- `"g" cannot move a record to "s"` / `records strand in "s"`: the desk + is missing a transition the state machine allows, or the state + machine is missing the edge. +- `action ... does not match any declared question id`: the page does + not exist. +- `entry page ... names no one responsible`: add `responsible`. diff --git a/intakes/contractor.yaml b/intakes/contractor.yaml new file mode 100644 index 0000000..405900b --- /dev/null +++ b/intakes/contractor.yaml @@ -0,0 +1,44 @@ +# An intake as the owner of a small building contractor would fill it +# in: plain words, no portal vocabulary. These are the fields the +# onboarding site asks for. +business: Nordvik Bygg +contact: { name: Ingrid Nordvik, email: ingrid@nordvikbygg.example } +language: en + +what_we_do: > + We are twelve people. We survey old buildings, draw up rehabilitation + plans, handle the permit application, and do the building work with + our own carpenters plus subcontractors for electrical, plumbing and + scaffolding. + +who_comes_to_us: > + Housing boards and private owners who have a problem with a building + and want someone to look at it. Owners who already know what they + want built and need drawings, a permit and a price. Trade firms who + want to be on our list of subcontractors. Carpenters and apprentices + asking if we are hiring. + +what_we_keep_track_of: > + Enquiries: they come in, we go and look at the building, we send a + quote, and then we either win it or lose it. Sometimes it is simply + not for us and we say so straight away. + Subcontractors: they apply, we check their company registration, + trade approval and insurance. Often papers are missing and we have to + ask again. Then they are approved or turned down. + Job applications: we read them, invite some to an interview, and hire + or decline. + Things that go wrong on site: a hazard, a near miss, wrong materials + delivered. Someone reports it, the safety officer decides what is + done about it, and closes it when it is fixed. + +who_is_involved: > + Two project managers handle enquiries, subcontractors and hiring. + One safety officer handles everything reported from site. + The crew, ours and the subcontractors' people, are the ones on site + who report things. They are on their phones, with gloves on, and + will not fill in a long form. + +what_slows_us_down: > + Enquiries arrive by phone, email and text and get lost. We never know + which subcontractors have valid papers. Site problems get mentioned + at lunch and forgotten. diff --git a/portal-portal/aggregates.yaml b/portal-portal/aggregates.yaml new file mode 100644 index 0000000..40f8021 --- /dev/null +++ b/portal-portal/aggregates.yaml @@ -0,0 +1,17 @@ +# One onboarding per business. `open` is the intake as submitted; the +# generator (portal-trainer generate) turns it into a draft content +# repo; the business reviews the draft and either it goes live or it +# comes back for another pass. +aggregates: + - bucket: onboardings + initial: open + states: + open: { event: intake_received } + drafted: { event: draft_generated } + revising: { event: revision_requested } + live: { event: went_live } + declined: { event: declined } + transitions: + open: [drafted, declined] + drafted: [live, revising, declined] + revising: [drafted, declined] diff --git a/portal-portal/needs.yaml b/portal-portal/needs.yaml new file mode 100644 index 0000000..fb30cdc --- /dev/null +++ b/portal-portal/needs.yaml @@ -0,0 +1,41 @@ +# The onboarding site is a portal site, so it is held to a needs spec +# like any other. +needs: + - id: onboard-business + who: someone who runs a business where requests, approvals and handovers get lost + wants: a working site for their own roles, without commissioning software + via: Describe your business and get a first draft + lands_in: onboardings + handled_by: owners + done_when: [live, declined] + max_steps: 1 + # The intake is the interview: two to reach them, four about the work. + max_fields: 6 + +personas: + - id: contractor-owner + need: onboard-business + text: > + I run a building firm. Enquiries come by phone, text and email + and we lose track of who promised what. I want one place where + they land and my project managers pick them up. + + - id: clinic-manager + need: onboard-business + text: > + We are a physiotherapy clinic with six therapists. Referrals, + cancellations and insurance approvals all go through one inbox + that only our receptionist understands. I'd like that sorted. + + - id: sports-club + need: onboard-business + text: > + I volunteer for a youth football club. Parents sign kids up, + coaches need to approve them, and the treasurer chases fees. + Right now it's a spreadsheet and a group chat. + + - id: sceptical-founder + need: onboard-business + text: > + I've been burned by agencies before. If I can explain my + workflow and see something real before paying anyone, I'm in. diff --git a/portal-portal/questions/desk/_section.yaml b/portal-portal/questions/desk/_section.yaml new file mode 100644 index 0000000..670fca7 --- /dev/null +++ b/portal-portal/questions/desk/_section.yaml @@ -0,0 +1 @@ +qualifies: owners diff --git a/portal-portal/questions/desk/index.yaml b/portal-portal/questions/desk/index.yaml new file mode 100644 index 0000000..602ec3d --- /dev/null +++ b/portal-portal/questions/desk/index.yaml @@ -0,0 +1,25 @@ +name: Which businesses are waiting on us? +responsible: + name: Bendik Aagaard Lynghaug + contact: bl@uhhm.no +alternatives: + - name: Onboardings + description: > + Run `portal-trainer generate` on an open intake, push the draft, + send the link, then mark it drafted. Revisions come back here. + action: /desk + consequence: [Confirm] + features: + - name: "" + description: "" + resource: + source: { kind: kv, bucket: onboardings } + requires_group: owners + transitions: + - { to: drafted, label: Draft sent } + - { to: declined, label: Decline } + - { from: drafted, to: live, label: Went live } + - { from: drafted, to: revising, label: Revision asked } + - { from: drafted, to: declined, label: Decline } + - { from: revising, to: drafted, label: Revised draft sent } + - { from: revising, to: declined, label: Decline } diff --git a/portal-portal/questions/how.yaml b/portal-portal/questions/how.yaml new file mode 100644 index 0000000..bcd1197 --- /dev/null +++ b/portal-portal/questions/how.yaml @@ -0,0 +1,21 @@ +name: How does an answer become a finished job? +description: > + Every page asks one question. Every answer lands with the person + whose job it is, and carries its own next steps. +alternatives: + - name: A visitor answers + description: > + They pick what they came for and fill in the few fields that + person needs. They are told who reads it and when. + - name: The right role sees it + description: > + Each role has a desk listing what is theirs, with one button per + step your process allows - quoted, approved, closed. + - name: Nothing waits unseen + description: > + Every case has a state and an owner, so what is open, and with + whom, is always one page away. + - name: Describe your business and get a first draft + description: Five answers in your own words. + action: / + consequence: [Start] diff --git a/portal-portal/questions/index.yaml b/portal-portal/questions/index.yaml new file mode 100644 index 0000000..8ff846f --- /dev/null +++ b/portal-portal/questions/index.yaml @@ -0,0 +1,49 @@ +name: What does your business keep doing by hand? +description: > + Tell us who comes to you, what you keep track of, and who handles + what. You get a site that asks each of those people the right + question and moves every answer to the person who acts on it. +responsible: + name: Bendik Aagaard Lynghaug + contact: bl@uhhm.no +alternatives: + - name: Describe your business and get a first draft + description: > + Five answers in your own words. We turn them into pages, forms + and a desk for each role, and send you a link to try it. + action: /received + consequence: [Send it] + record_as: onboardings + encouragements: + - Write it the way you would explain it to a new colleague. + features: + - name: You + requirements: + - { name: business, label: Business name } + - { name: email, label: Your email, type: email } + - name: The work + description: Plain words are best. Stages, handovers and waiting are what we listen for. + requirements: + - name: what_we_do + label: What you do + type: textarea + - name: who_comes_to_us + label: Who comes to you, and what each of them wants + type: textarea + - name: what_we_keep_track_of + label: What you keep track of, and the stages it goes through + type: textarea + - name: who_is_involved + label: Who on your side handles what + type: textarea + - name: what_slows_us_down + label: Where things get lost or wait today + type: textarea + optional: true + + - name: See how a question becomes a finished job + description: > + One enquiry followed from the visitor's answer to the desk that + closes it. + action: /how + consequence: [Show me] diff --git a/portal-portal/questions/received.yaml b/portal-portal/questions/received.yaml new file mode 100644 index 0000000..6d125a6 --- /dev/null +++ b/portal-portal/questions/received.yaml @@ -0,0 +1,15 @@ +followup: true +name: What happens with what you sent? +description: > + Bendik reads every intake. Within two working days you get a link to + a working draft built from your answers. +alternatives: + - name: You try the draft as each of your roles + description: > + Click through it as a customer, then as the colleague who handles + that customer. Tell us what reads wrong and the draft comes back + revised. + - name: Back to the start + description: "" + action: / + consequence: [Back] diff --git a/portal-portal/site.yaml b/portal-portal/site.yaml new file mode 100644 index 0000000..f05560b --- /dev/null +++ b/portal-portal/site.yaml @@ -0,0 +1,2 @@ +title: Portal +lang: en diff --git a/runs/uhhm-2026-09-21/ollama/ans-full-granite4.1.jsonl b/runs/uhhm-2026-09-21/ollama/ans-full-granite4.1.jsonl new file mode 100644 index 0000000..b78e9f4 --- /dev/null +++ b/runs/uhhm-2026-09-21/ollama/ans-full-granite4.1.jsonl @@ -0,0 +1,8 @@ +{"id":"architecture-firm@/","choice":"Work happens through dialogue, not tickets","confidence":0.95} +{"id":"climbing-gym-founder@/","choice":"Good software begins with a clear argument","confidence":0.98} +{"id":"municipality-buyer@/","choice":"Good software begins with a clear argument","confidence":0.97} +{"id":"exhibition-visitor@/","choice":"Attention is earned","confidence":0.9} +{"id":"tech-journalist@/","choice":"Attention is earned","confidence":0.92} +{"id":"rust-developer@/","choice":"Work happens through dialogue, not tickets","confidence":0.96} +{"id":"interaction-designer@/","choice":"Work happens through dialogue, not tickets","confidence":0.94} +{"id":"student-intern@/","choice":"Work happens through dialogue, not tickets","confidence":0.88} diff --git a/runs/uhhm-2026-09-21/ollama/ans-full-qwen3.jsonl b/runs/uhhm-2026-09-21/ollama/ans-full-qwen3.jsonl new file mode 100644 index 0000000..53c02f2 --- /dev/null +++ b/runs/uhhm-2026-09-21/ollama/ans-full-qwen3.jsonl @@ -0,0 +1,8 @@ +{"id":"architecture-firm@/","choice":"Good software begins with a clear argument","confidence":0.95} +{"id":"climbing-gym-founder@/","choice":"Work happens through dialogue, not tickets","confidence":0.95} +{"id":"municipality-buyer@/","choice":"Good software begins with a clear argument","confidence":0.95} +{"id":"exhibition-visitor@/","choice":"Attention is earned","confidence":0.95} +{"id":"tech-journalist@/","choice":"Attention is earned","confidence":0.95} +{"id":"rust-developer@/","choice":"Work happens through dialogue, not tickets","confidence":0.95} +{"id":"interaction-designer@/","choice":"Work happens through dialogue, not tickets","confidence":0.95} +{"id":"student-intern@/","choice":"Work happens through dialogue, not tickets","confidence":0.95} diff --git a/runs/uhhm-2026-09-21/ollama/ans-skim-granite4.1.jsonl b/runs/uhhm-2026-09-21/ollama/ans-skim-granite4.1.jsonl new file mode 100644 index 0000000..5ebca4a --- /dev/null +++ b/runs/uhhm-2026-09-21/ollama/ans-skim-granite4.1.jsonl @@ -0,0 +1,8 @@ +{"id":"architecture-firm@/","choice":"Work happens through dialogue, not tickets","confidence":0.95} +{"id":"climbing-gym-founder@/","choice":"Good software begins with a clear argument","confidence":0.9} +{"id":"municipality-buyer@/","choice":"Good software begins with a clear argument","confidence":0.93} +{"id":"exhibition-visitor@/","choice":"Attention is earned","confidence":0.96} +{"id":"tech-journalist@/","choice":"Attention is earned","confidence":0.94} +{"id":"rust-developer@/","choice":"Work happens through dialogue, not tickets","confidence":0.92} +{"id":"interaction-designer@/","choice":"Good software begins with a clear argument","confidence":0.91} +{"id":"student-intern@/","choice":"Work happens through dialogue, not tickets","confidence":0.88} diff --git a/runs/uhhm-2026-09-21/ollama/ans-skim-qwen3.jsonl b/runs/uhhm-2026-09-21/ollama/ans-skim-qwen3.jsonl new file mode 100644 index 0000000..880c1e2 --- /dev/null +++ b/runs/uhhm-2026-09-21/ollama/ans-skim-qwen3.jsonl @@ -0,0 +1,8 @@ +{"id":"architecture-firm@/","choice":"Good software begins with a clear argument","confidence":0.95} +{"id":"climbing-gym-founder@/","choice":"Work happens through dialogue, not tickets","confidence":0.85} +{"id":"municipality-buyer@/","choice":"Good software begins with a clear argument","confidence":0.90} +{"id":"exhibition-visitor@/","choice":"Attention is earned","confidence":0.80} +{"id":"tech-journalist@/","choice":"Attention is earned","confidence":0.85} +{"id":"rust-developer@/","choice":"Work happens through dialogue, not tickets","confidence":0.80} +{"id":"interaction-designer@/","choice":"Work happens through dialogue, not tickets","confidence":0.90} +{"id":"student-intern@/","choice":"Work happens through dialogue, not tickets","confidence":0.75} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..5c88ad5 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,392 @@ +//! Generates a portal content repo from a business intake. +//! +//! Three phases, each held to portal's `question_lint`: +//! +//! 1. **Needs.** The intake (what the business does, who comes to it, +//! what it keeps track of and through which stages, who handles +//! what) becomes `needs.yaml`. After this phase the file is frozen: +//! it is the spec, and the generator may not weaken it to pass. +//! 2. **Structure.** Pages, `aggregates.yaml` and desks are generated +//! and revised against the lint's findings until it exits zero. +//! 3. **Wording.** The personas are answered blind (headings only, +//! then with descriptions) and misroutes are fed back as wording +//! defects. Only pages may change. +//! +//! The LLM is an external command: prompt on stdin, text on stdout. +//! `claude -p` works; so does anything else with that shape. + +use std::collections::BTreeMap; +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use anyhow::{anyhow, bail, Context}; + +const REFERENCE: &str = include_str!("../generator/reference.md"); + +struct Opts { + intake: PathBuf, + out: PathBuf, + lint: String, + llm: String, + rounds: usize, + wording_rounds: usize, +} + +fn main() -> anyhow::Result<()> { + // `answer --tasks [--llm ]`: the blind answerer alone, + // for grading an existing site with any backend. Answers go to + // stdout, ready for `question_lint --needs-score`. + let argv: Vec = std::env::args().skip(1).collect(); + if argv.first().map(String::as_str) == Some("answer") { + let value = |flag: &str| { + argv.iter() + .position(|a| a == flag) + .and_then(|i| argv.get(i + 1)) + .cloned() + }; + let tasks = value("--tasks").ok_or_else(|| anyhow!("usage: portal-trainer answer --tasks [--llm 'cmd']"))?; + let cmd = value("--llm").unwrap_or_else(|| "claude -p".to_string()); + print!("{}", answer_blind(&cmd, &std::fs::read_to_string(&tasks).with_context(|| format!("reading {tasks}"))?)?); + return Ok(()); + } + let opts = parse_args()?; + let intake = std::fs::read_to_string(&opts.intake) + .with_context(|| format!("reading {}", opts.intake.display()))?; + std::fs::create_dir_all(opts.out.join("questions"))?; + let mut log = Vec::new(); + + // Phase 1: the spec. + let mut feedback = String::new(); + for round in 1..=opts.rounds { + let prompt = format!( + "{REFERENCE}\n\n# Task\n\nA business filled in this intake. Write ONLY `needs.yaml` for it: every need the intake implies, for the public and for each role inside the business, with states taken from how the business says things progress, and personas.\n\n## Intake\n\n{intake}\n{feedback}\n{FORMAT}" + ); + let files = parse_files(&llm(&opts.llm, &prompt)?)?; + let Some(needs) = files.get("needs.yaml") else { + feedback = "\n## Your last attempt\n\nIt contained no needs.yaml.\n".to_string(); + continue; + }; + std::fs::write(opts.out.join("needs.yaml"), needs)?; + // A placeholder front door, so the lint gets as far as parsing + // the spec; every need fails, a parse error is what we look for. + std::fs::write(opts.out.join("questions/index.yaml"), "name: Placeholder?\n")?; + let (_, report) = lint(&opts, &[])?; + if let Some(line) = report.lines().find(|l| l.contains("parsing needs.yaml")) { + eprintln!("needs round {round}: {line}"); + feedback = format!("\n## Your last attempt failed to parse\n\n{line}\n\n```yaml\n{needs}\n```\n"); + continue; + } + eprintln!("needs round {round}: needs.yaml parses, frozen"); + log.push(format!("needs: parsed in round {round}")); + break; + } + let needs = std::fs::read_to_string(opts.out.join("needs.yaml")).context("no needs.yaml was produced")?; + + // Phase 2: structure. + let mut green = false; + let mut report = String::new(); + for round in 1..=opts.rounds { + let prompt = if round == 1 { + format!( + "{REFERENCE}\n\n# Task\n\nWrite the content repo that meets this frozen `needs.yaml`: `aggregates.yaml`, `site.yaml`, and every page under `questions/` - the front door, followup pages, role-gated pages, one desk per handling group. Do not return needs.yaml.\n\n## Intake\n\n{intake}\n\n## needs.yaml (frozen)\n\n```yaml\n{needs}\n```\n\n{FORMAT}" + ) + } else { + format!( + "{REFERENCE}\n\n# Task\n\nThe verifier rejected the repo below. Fix it. Return only the files you change, each complete. To remove a file, return it with the single line DELETE. needs.yaml is frozen; do not return it.\n\n## Verifier output\n\n```\n{report}\n```\n\n## needs.yaml (frozen)\n\n```yaml\n{needs}\n```\n\n## Current files\n\n{}\n\n{FORMAT}", + dump(&opts.out)? + ) + }; + apply(&opts.out, parse_files(&llm(&opts.llm, &prompt)?)?)?; + let (ok, r) = lint(&opts, &[])?; + report = r; + let fails = report.lines().filter(|l| l.starts_with("FAIL")).count(); + eprintln!("structure round {round}: {}", if ok { "green".to_string() } else { format!("{fails} FAIL line(s)") }); + log.push(format!("structure round {round}: ok={ok} fails={fails}")); + if ok { + green = true; + break; + } + } + if !green { + std::fs::write(opts.out.join("run.log"), format!("{}\n\n{report}", log.join("\n")))?; + bail!("structure never went green in {} round(s); last verifier output is in run.log", opts.rounds); + } + + // Phase 3: wording. + for round in 0..=opts.wording_rounds { + let mut defects = String::new(); + let mut summary = Vec::new(); + for skim in [true, false] { + let tasks_path = opts.out.join(if skim { "tasks-skim.jsonl" } else { "tasks.jsonl" }); + let mut args = vec!["--needs-tasks".to_string(), tasks_path.display().to_string()]; + if skim { + args.push("--skim".to_string()); + } + lint(&opts, &args)?; + let answers = answer_blind(&opts.llm, &std::fs::read_to_string(&tasks_path)?)?; + let answers_path = opts.out.join(if skim { "answers-skim.jsonl" } else { "answers.jsonl" }); + std::fs::write(&answers_path, answers)?; + let mut args = vec!["--needs-score".to_string(), answers_path.display().to_string()]; + if skim { + args.push("--skim".to_string()); + } + let (_, scored) = lint(&opts, &args)?; + let mode = if skim { "headings only" } else { "headings and descriptions" }; + for line in scored.lines().filter(|l| l.starts_with("MISROUTE") || l.starts_with("HESITANT")) { + defects.push_str(&format!("[{mode}] {line}\n")); + } + if let Some(line) = scored.lines().find(|l| l.starts_with("wording:")) { + summary.push(format!("{mode}: {line}")); + } + } + eprintln!("wording round {round}: {}", summary.join(" | ")); + log.push(format!("wording round {round}: {}", summary.join(" | "))); + if defects.is_empty() || round == opts.wording_rounds { + break; + } + let personas = needs.split("\npersonas:").nth(1).unwrap_or_default(); + let prompt = format!( + "{REFERENCE}\n\n# Task\n\nThe structure is correct. Visitors are picking the wrong alternative. Each MISROUTE below is a persona who, reading the page, chose an alternative that does not meet their need; HESITANT means they chose right and nearly did not. Rewrite alternative headings first (a heading that needs its description has failed), descriptions second. Keep every `record_as`, `action`, field and state as it is. If you rename an alternative that needs.yaml names in `via`, you may not: keep that one. Return only the pages you change, each complete.\n\n## Defects\n\n```\n{defects}```\n\n## The personas\n\n```yaml\npersonas:{personas}\n```\n\n## Current files\n\n{}\n\n{FORMAT}", + dump(&opts.out)? + ); + let before = snapshot(&opts.out)?; + apply(&opts.out, parse_files(&llm(&opts.llm, &prompt)?)?)?; + let (ok, r) = lint(&opts, &[])?; + if !ok { + eprintln!("wording round {round}: the rewrite broke structure, reverted"); + log.push(format!("wording round {round}: rewrite reverted\n{r}")); + restore(&opts.out, before)?; + break; + } + } + std::fs::write(opts.out.join("run.log"), log.join("\n") + "\n")?; + println!("content repo written to {}", opts.out.display()); + Ok(()) +} + +const FORMAT: &str = "# Output format\n\nReturn files and nothing else, each as:\n\n<<>>\n...complete file content...\n<<>>\n\nNo prose, no code fences."; + +fn parse_args() -> anyhow::Result { + let mut args = std::env::args().skip(1); + if args.next().as_deref() != Some("generate") { + bail!("usage: portal-trainer generate --intake --out [--lint question_lint] [--llm 'claude -p'] [--rounds 4] [--wording-rounds 2]"); + } + let mut opts = Opts { + intake: PathBuf::new(), + out: PathBuf::new(), + lint: "question_lint".to_string(), + llm: "claude -p".to_string(), + rounds: 4, + wording_rounds: 2, + }; + while let Some(arg) = args.next() { + let mut value = || args.next().ok_or_else(|| anyhow!("{arg} needs a value")); + match arg.as_str() { + "--intake" => opts.intake = value()?.into(), + "--out" => opts.out = value()?.into(), + "--lint" => opts.lint = value()?, + "--llm" => opts.llm = value()?, + "--rounds" => opts.rounds = value()?.parse()?, + "--wording-rounds" => opts.wording_rounds = value()?.parse()?, + other => bail!("unknown argument {other}"), + } + } + if opts.intake.as_os_str().is_empty() || opts.out.as_os_str().is_empty() { + bail!("--intake and --out are required"); + } + Ok(opts) +} + +/// Prompt on stdin, text on stdout. +fn llm(cmd: &str, prompt: &str) -> anyhow::Result { + let mut child = Command::new("sh") + .arg("-c") + .arg(cmd) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .with_context(|| format!("starting {cmd:?}"))?; + let mut stdin = child.stdin.take().expect("piped"); + let prompt = prompt.to_string(); + // Written from a thread: a large prompt would otherwise deadlock + // against a child that starts writing before it finishes reading. + let writer = std::thread::spawn(move || stdin.write_all(prompt.as_bytes())); + let output = child.wait_with_output()?; + writer.join().ok(); + if !output.status.success() { + bail!("{cmd:?} exited with {}", output.status); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +fn lint(opts: &Opts, extra: &[String]) -> anyhow::Result<(bool, String)> { + let output = Command::new(&opts.lint) + .arg("--path") + .arg(opts.out.join("questions")) + .args(extra) + .output() + .with_context(|| format!("running {}", opts.lint))?; + let mut report = String::from_utf8_lossy(&output.stdout).into_owned(); + report.push_str(&String::from_utf8_lossy(&output.stderr)); + Ok((output.status.success(), report)) +} + +/// `<<>> ... <<>>` blocks. Anything outside a block is +/// ignored, so a model that adds a sentence does no harm. +fn parse_files(text: &str) -> anyhow::Result> { + let mut files = BTreeMap::new(); + let mut current: Option<(String, String)> = None; + for line in text.lines() { + if let Some(path) = line.trim().strip_prefix("<<>>")) { + current = Some((path.trim().to_string(), String::new())); + } else if line.trim() == "<<>>" { + if let Some((path, body)) = current.take() { + files.insert(path, body); + } + } else if let Some((_, body)) = current.as_mut() { + body.push_str(line); + body.push('\n'); + } + } + if files.is_empty() { + bail!("the model returned no <<>> blocks"); + } + Ok(files) +} + +/// Model output is written to disk, so the path is checked, not +/// trusted: repo-relative, yaml, and only where content lives. +fn allowed(path: &str) -> bool { + let p = Path::new(path); + let plain = p.components().all(|c| matches!(c, std::path::Component::Normal(_))); + plain + && p.extension().and_then(|e| e.to_str()) == Some("yaml") + && (matches!(path, "aggregates.yaml" | "site.yaml") || path.starts_with("questions/")) +} + +fn apply(out: &Path, files: BTreeMap) -> anyhow::Result<()> { + for (path, body) in files { + if !allowed(&path) { + eprintln!("ignored {path:?}: outside what the generator may write"); + continue; + } + let target = out.join(&path); + if body.trim() == "DELETE" { + std::fs::remove_file(&target).ok(); + continue; + } + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(target, body)?; + } + Ok(()) +} + +fn content_files(out: &Path) -> anyhow::Result> { + fn walk(dir: &Path, found: &mut Vec) -> anyhow::Result<()> { + for entry in std::fs::read_dir(dir)? { + let path = entry?.path(); + if path.is_dir() { + walk(&path, found)?; + } else if path.extension().and_then(|e| e.to_str()) == Some("yaml") { + found.push(path); + } + } + Ok(()) + } + let mut found = Vec::new(); + for name in ["aggregates.yaml", "site.yaml"] { + if out.join(name).exists() { + found.push(out.join(name)); + } + } + walk(&out.join("questions"), &mut found)?; + found.sort(); + Ok(found) +} + +fn snapshot(out: &Path) -> anyhow::Result> { + content_files(out)? + .into_iter() + .map(|p| Ok((p.clone(), std::fs::read_to_string(p)?))) + .collect() +} + +fn restore(out: &Path, before: Vec<(PathBuf, String)>) -> anyhow::Result<()> { + for path in content_files(out)? { + std::fs::remove_file(path)?; + } + for (path, body) in before { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(path, body)?; + } + Ok(()) +} + +fn dump(out: &Path) -> anyhow::Result { + let mut text = String::new(); + for path in content_files(out)? { + let rel = path.strip_prefix(out).unwrap_or(&path).display().to_string(); + text.push_str(&format!("<<>>\n{}<<>>\n\n", std::fs::read_to_string(&path)?)); + } + Ok(text) +} + +/// The answering side never sees `expect` or which need a persona has. +fn answer_blind(cmd: &str, tasks: &str) -> anyhow::Result { + let mut blind = String::new(); + for line in tasks.lines().filter(|l| !l.trim().is_empty()) { + let mut task: serde_json::Value = serde_json::from_str(line)?; + if let Some(map) = task.as_object_mut() { + map.remove("expect"); + map.remove("need"); + } + blind.push_str(&serde_json::to_string(&task)?); + blind.push('\n'); + } + if blind.is_empty() { + return Ok(String::new()); + } + let prompt = format!( + "You are a typed-decision answering engine. Each line below is a JSON task: `state.body` is a website visitor describing their situation in their own words; `questions.pick` holds `instructions` and `criteria`, a map from option label to what that option says.\n\nFor each task decide which single option that visitor would most plausibly choose, judging only from their text and the option wording, and give your honest confidence between 0 and 1. If two options are similarly plausible, confidence is low.\n\nReply with ONLY JSONL, one line per task, same order:\n{{\"id\":\"\",\"choice\":\"