One binary: the lint and the trainer are commands of portal
Test / test (pull_request) Successful in 1m7s
Test / test (pull_request) Successful in 1m7s
portal serve runs a site; portal lint is what iris was; portal plan, build, report and the rest are what portal-trainer was. One source at one version, so the three never disagree about what content means. Both histories are merged in (subtree). The release attaches the bare binary for a content repo's CI. The trainer's plans, intakes, engines and reference prompt come along; its runs stay in the old repo. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GT37Z1Xtfd9pUuQtMTg6Yt
This commit is contained in:
Bendik Aagaard Lynghaug
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
d0d1cbf23d
commit
96b3746643
@@ -0,0 +1,593 @@
|
||||
//! Generates a portal content repo from a business intake.
|
||||
//!
|
||||
//! Three phases, each held to portal's `question_lint`:
|
||||
//!
|
||||
//! 1. **Needs.** The intake (what the business does, who comes to it,
|
||||
//! what it keeps track of and through which stages, who handles
|
||||
//! what) becomes `needs.yaml`. After this phase the file is frozen:
|
||||
//! it is the spec, and the generator may not weaken it to pass.
|
||||
//! 2. **Structure.** Pages, `aggregates.yaml` and desks are generated
|
||||
//! and revised against the lint's findings until it exits zero.
|
||||
//! 3. **Wording.** The personas are answered blind (headings only,
|
||||
//! then with descriptions) and misroutes are fed back as wording
|
||||
//! defects. Only pages may change.
|
||||
//!
|
||||
//! The LLM is an external command: prompt on stdin, text on stdout.
|
||||
//! `claude -p` works; so does anything else with that shape.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::io::Write as _;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
use anyhow::{anyhow, bail, Context};
|
||||
|
||||
pub mod build;
|
||||
mod card;
|
||||
pub mod plan;
|
||||
mod rate;
|
||||
mod report;
|
||||
mod score;
|
||||
mod simulate;
|
||||
|
||||
pub const REFERENCE: &str = include_str!("reference.md");
|
||||
|
||||
struct Opts {
|
||||
intake: PathBuf,
|
||||
out: PathBuf,
|
||||
lint: String,
|
||||
llm: String,
|
||||
rounds: usize,
|
||||
wording_rounds: usize,
|
||||
/// The model that plays the visitor. A generator grading its own
|
||||
/// wording agrees with itself, so this should be a different one.
|
||||
visitor: Option<String>,
|
||||
}
|
||||
|
||||
/// The trainer, as `portal <command> ...`: `argv` is everything after
|
||||
/// `portal`. `answer --tasks <file> [--llm <cmd>]` is the blind
|
||||
/// answerer alone, for grading an existing site with any backend;
|
||||
/// answers go to stdout, ready for `portal lint --needs-score`.
|
||||
pub fn run(argv: Vec<String>) -> anyhow::Result<()> {
|
||||
match argv.first().map(String::as_str) {
|
||||
Some("simulate") => return simulate::run(&argv),
|
||||
Some("compare") => return simulate::compare(&argv),
|
||||
Some("plan") => return plan::run(&argv),
|
||||
Some("build") => return build::run(&argv),
|
||||
Some("rate") => return rate::rate(&argv),
|
||||
Some("report") => return report::run(&argv),
|
||||
Some("prefer") => return rate::prefer(&argv),
|
||||
Some("score") => return score::run(&argv),
|
||||
_ => {}
|
||||
}
|
||||
if argv.first().map(String::as_str) == Some("answer") {
|
||||
let value = |flag: &str| {
|
||||
argv.iter()
|
||||
.position(|a| a == flag)
|
||||
.and_then(|i| argv.get(i + 1))
|
||||
.cloned()
|
||||
};
|
||||
let tasks = value("--tasks").ok_or_else(|| anyhow!("usage: portal-trainer answer --tasks <tasks.jsonl> [--repo <content dir>] [--llm 'cmd'] [--seed <n>]"))?;
|
||||
let cmd = value("--llm").unwrap_or_else(|| "claude -p".to_string());
|
||||
let repo = value("--repo").map(PathBuf::from);
|
||||
// --seed: the same tasks answered again with other shuffles and
|
||||
// sampling, so a score comes with its spread. 0 is the run
|
||||
// every earlier score used.
|
||||
let seed: u64 = value("--seed").and_then(|s| s.parse().ok()).unwrap_or(0);
|
||||
print!("{}", answer_blind(&cmd, &std::fs::read_to_string(&tasks).with_context(|| format!("reading {tasks}"))?, repo.as_deref(), seed)?);
|
||||
return Ok(());
|
||||
}
|
||||
let opts = parse_args()?;
|
||||
let intake = std::fs::read_to_string(&opts.intake)
|
||||
.with_context(|| format!("reading {}", opts.intake.display()))?;
|
||||
std::fs::create_dir_all(opts.out.join("questions"))?;
|
||||
std::fs::create_dir_all(opts.out.join("llm"))?;
|
||||
LLM_LOG.set(opts.out.join("llm")).ok();
|
||||
let mut log = Vec::new();
|
||||
|
||||
// Phase 1: the spec.
|
||||
let mut feedback = String::new();
|
||||
for round in 1..=opts.rounds {
|
||||
let prompt = format!(
|
||||
"{REFERENCE}\n\n# Task\n\nA business filled in this intake. Write ONLY `needs.yaml` for it: every need the intake implies, for the public and for each role inside the business, with states taken from how the business says things progress, and personas.\n\n## Intake\n\n{intake}\n{feedback}\n{FORMAT}"
|
||||
);
|
||||
let Ok(files) = parse_files(&llm(&opts.llm, &prompt)?) else {
|
||||
eprintln!("needs round {round}: the reply had no file blocks, asking again");
|
||||
feedback = "\n## Your last attempt\n\nIt contained no <<<FILE ...>>> blocks at all. Return needs.yaml in exactly that format.\n".to_string();
|
||||
continue;
|
||||
};
|
||||
let Some(needs) = files.get("needs.yaml") else {
|
||||
feedback = "\n## Your last attempt\n\nIt contained no needs.yaml.\n".to_string();
|
||||
continue;
|
||||
};
|
||||
std::fs::write(opts.out.join("needs.yaml"), needs)?;
|
||||
// A placeholder front door, so the lint gets as far as parsing
|
||||
// the spec; every need fails, a parse error is what we look for.
|
||||
std::fs::write(opts.out.join("questions/index.yaml"), "name: Placeholder?\n")?;
|
||||
let (_, report) = lint(&opts, &[])?;
|
||||
if let Some(line) = report.lines().find(|l| l.contains("parsing needs.yaml")) {
|
||||
eprintln!("needs round {round}: {line}");
|
||||
feedback = format!("\n## Your last attempt failed to parse\n\n{line}\n\n```yaml\n{needs}\n```\n");
|
||||
continue;
|
||||
}
|
||||
eprintln!("needs round {round}: needs.yaml parses, frozen");
|
||||
log.push(format!("needs: parsed in round {round}"));
|
||||
feedback.clear();
|
||||
break;
|
||||
}
|
||||
if !feedback.is_empty() {
|
||||
bail!("needs.yaml never parsed in {} round(s); the replies are in {}/llm/", opts.rounds, opts.out.display());
|
||||
}
|
||||
let needs = std::fs::read_to_string(opts.out.join("needs.yaml")).context("no needs.yaml was produced")?;
|
||||
|
||||
// Phase 2: structure. The page-writing model sees the needs and
|
||||
// never the personas: pages written with the test in view route
|
||||
// their own personas perfectly and strangers badly (measured:
|
||||
// 10/10 on its own personas, 8/10 headings-only on held-out ones).
|
||||
let needs_full = needs;
|
||||
let needs = needs_full
|
||||
.split("\npersonas:")
|
||||
.next()
|
||||
.unwrap_or(&needs_full)
|
||||
.to_string();
|
||||
let mut green = false;
|
||||
let mut report = String::new();
|
||||
for round in 1..=opts.rounds {
|
||||
let prompt = if round == 1 {
|
||||
format!(
|
||||
"{REFERENCE}\n\n# Task\n\nWrite the content repo that meets this frozen `needs.yaml`: `aggregates.yaml`, `site.yaml`, and every page under `questions/` - the front door, followup pages, role-gated pages, one desk per handling group. Do not return needs.yaml.\n\n## Intake\n\n{intake}\n\n## needs.yaml (frozen)\n\n```yaml\n{needs}\n```\n\n{FORMAT}"
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"{REFERENCE}\n\n# Task\n\nThe verifier rejected the repo below. Fix it. Return only the files you change, each complete. To remove a file, return it with the single line DELETE. needs.yaml is frozen; do not return it.\n\n## Verifier output\n\n```\n{report}\n```\n\n## needs.yaml (frozen)\n\n```yaml\n{needs}\n```\n\n## Current files\n\n{}\n\n{FORMAT}",
|
||||
dump(&opts.out)?
|
||||
)
|
||||
};
|
||||
match parse_files(&llm(&opts.llm, &prompt)?) {
|
||||
Ok(files) => apply(&opts.out, files)?,
|
||||
Err(_) => eprintln!("structure round {round}: the reply had no file blocks"),
|
||||
}
|
||||
let (ok, r) = lint(&opts, &[])?;
|
||||
report = r;
|
||||
let fails = report.lines().filter(|l| l.starts_with("FAIL")).count();
|
||||
eprintln!("structure round {round}: {}", if ok { "green".to_string() } else { format!("{fails} FAIL line(s)") });
|
||||
log.push(format!("structure round {round}: ok={ok} fails={fails}"));
|
||||
if ok {
|
||||
green = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !green {
|
||||
std::fs::write(opts.out.join("run.log"), format!("{}\n\n{report}", log.join("\n")))?;
|
||||
bail!("structure never went green in {} round(s); last verifier output is in run.log", opts.rounds);
|
||||
}
|
||||
|
||||
// Phase 3: wording.
|
||||
for round in 0..=opts.wording_rounds {
|
||||
let mut defects = String::new();
|
||||
let mut summary = Vec::new();
|
||||
for skim in [true, false] {
|
||||
let tasks_path = opts.out.join(if skim { "tasks-skim.jsonl" } else { "tasks.jsonl" });
|
||||
let mut args = vec!["--needs-tasks".to_string(), tasks_path.display().to_string()];
|
||||
if skim {
|
||||
args.push("--skim".to_string());
|
||||
}
|
||||
lint(&opts, &args)?;
|
||||
let answers = answer_blind(opts.visitor.as_deref().unwrap_or(&opts.llm), &std::fs::read_to_string(&tasks_path)?, Some(&opts.out), 0)?;
|
||||
let answers_path = opts.out.join(if skim { "answers-skim.jsonl" } else { "answers.jsonl" });
|
||||
std::fs::write(&answers_path, answers)?;
|
||||
let mut args = vec!["--needs-score".to_string(), answers_path.display().to_string()];
|
||||
if skim {
|
||||
args.push("--skim".to_string());
|
||||
}
|
||||
let (_, scored) = lint(&opts, &args)?;
|
||||
let mode = if skim { "headings only" } else { "headings and descriptions" };
|
||||
for line in scored.lines().filter(|l| l.starts_with("MISROUTE") || l.starts_with("HESITANT")) {
|
||||
defects.push_str(&format!("[{mode}] {line}\n"));
|
||||
}
|
||||
if let Some(line) = scored.lines().find(|l| l.starts_with("wording:")) {
|
||||
summary.push(format!("{mode}: {line}"));
|
||||
}
|
||||
}
|
||||
eprintln!("wording round {round}: {}", summary.join(" | "));
|
||||
log.push(format!("wording round {round}: {}", summary.join(" | ")));
|
||||
if defects.is_empty() || round == opts.wording_rounds {
|
||||
break;
|
||||
}
|
||||
let personas = needs_full.split("\npersonas:").nth(1).unwrap_or_default();
|
||||
let prompt = format!(
|
||||
"{REFERENCE}\n\n# Task\n\nThe structure is correct. Visitors are picking the wrong alternative. Each MISROUTE below is a persona who, reading the page, chose an alternative that does not meet their need; HESITANT means they chose right and nearly did not. Rewrite alternative headings first (a heading that needs its description has failed), descriptions second. Keep every `record_as`, `action`, field and state as it is. If you rename an alternative that needs.yaml names in `via`, you may not: keep that one. Return only the pages you change, each complete.\n\n## Defects\n\n```\n{defects}```\n\n## The personas\n\n```yaml\npersonas:{personas}\n```\n\n## Current files\n\n{}\n\n{FORMAT}",
|
||||
dump(&opts.out)?
|
||||
);
|
||||
let before = snapshot(&opts.out)?;
|
||||
apply(&opts.out, parse_files(&llm(&opts.llm, &prompt)?)?)?;
|
||||
let (ok, r) = lint(&opts, &[])?;
|
||||
if !ok {
|
||||
eprintln!("wording round {round}: the rewrite broke structure, reverted");
|
||||
log.push(format!("wording round {round}: rewrite reverted\n{r}"));
|
||||
restore(&opts.out, before)?;
|
||||
break;
|
||||
}
|
||||
}
|
||||
std::fs::write(opts.out.join("run.log"), log.join("\n") + "\n")?;
|
||||
println!("content repo written to {}", opts.out.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const FORMAT: &str = "# Output format\n\nReturn files and nothing else, each as:\n\n<<<FILE relative/path.yaml>>>\n...complete file content...\n<<<END>>>\n\nNo prose, no code fences.";
|
||||
|
||||
fn parse_args() -> anyhow::Result<Opts> {
|
||||
let mut args = std::env::args().skip(1);
|
||||
if args.next().as_deref() != Some("generate") {
|
||||
bail!("usage: portal-trainer generate --intake <file> --out <dir> [--lint question_lint] [--llm 'claude -p'] [--rounds 4] [--wording-rounds 2]");
|
||||
}
|
||||
let mut opts = Opts {
|
||||
intake: PathBuf::new(),
|
||||
out: PathBuf::new(),
|
||||
lint: "question_lint".to_string(),
|
||||
llm: "claude -p".to_string(),
|
||||
rounds: 4,
|
||||
wording_rounds: 2,
|
||||
visitor: None,
|
||||
};
|
||||
while let Some(arg) = args.next() {
|
||||
let mut value = || args.next().ok_or_else(|| anyhow!("{arg} needs a value"));
|
||||
match arg.as_str() {
|
||||
"--intake" => opts.intake = value()?.into(),
|
||||
"--out" => opts.out = value()?.into(),
|
||||
"--lint" => opts.lint = value()?,
|
||||
"--llm" => opts.llm = value()?,
|
||||
"--rounds" => opts.rounds = value()?.parse()?,
|
||||
"--wording-rounds" => opts.wording_rounds = value()?.parse()?,
|
||||
"--visitor-llm" => opts.visitor = Some(value()?),
|
||||
other => bail!("unknown argument {other}"),
|
||||
}
|
||||
}
|
||||
if opts.intake.as_os_str().is_empty() || opts.out.as_os_str().is_empty() {
|
||||
bail!("--intake and --out are required");
|
||||
}
|
||||
Ok(opts)
|
||||
}
|
||||
|
||||
/// Prompt on stdin, text on stdout.
|
||||
/// Where `generate` keeps every prompt and reply (`<out>/llm/`). A
|
||||
/// model that returns nothing usable has to be read, not guessed at.
|
||||
static LLM_LOG: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
|
||||
static LLM_CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
fn llm(cmd: &str, prompt: &str) -> anyhow::Result<String> {
|
||||
llm_env(cmd, prompt, &[])
|
||||
}
|
||||
|
||||
/// As `llm`, with hints for the backend in its environment.
|
||||
fn llm_env(cmd: &str, prompt: &str, env: &[(&str, &str)]) -> anyhow::Result<String> {
|
||||
let reply = llm_with(cmd, prompt, env);
|
||||
if let Some(dir) = LLM_LOG.get() {
|
||||
let n = LLM_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
|
||||
std::fs::write(dir.join(format!("{n:02}-prompt.md")), prompt).ok();
|
||||
let text = match &reply {
|
||||
Ok(text) => text.clone(),
|
||||
Err(e) => format!("ERROR: {e}"),
|
||||
};
|
||||
std::fs::write(dir.join(format!("{n:02}-reply.md")), text).ok();
|
||||
}
|
||||
reply
|
||||
}
|
||||
|
||||
/// As `llm`, with hints for the backend in its environment (`SEED`,
|
||||
/// `FORMAT=json`, `TEMPERATURE`). A backend is free to ignore them.
|
||||
pub(crate) fn llm_with(cmd: &str, prompt: &str, env: &[(&str, &str)]) -> anyhow::Result<String> {
|
||||
let mut child = Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(cmd)
|
||||
.envs(env.iter().copied())
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.with_context(|| format!("starting {cmd:?}"))?;
|
||||
let mut stdin = child.stdin.take().expect("piped");
|
||||
let prompt = prompt.to_string();
|
||||
// Written from a thread: a large prompt would otherwise deadlock
|
||||
// against a child that starts writing before it finishes reading.
|
||||
let writer = std::thread::spawn(move || stdin.write_all(prompt.as_bytes()));
|
||||
let output = child.wait_with_output()?;
|
||||
writer.join().ok();
|
||||
if !output.status.success() {
|
||||
bail!("{cmd:?} exited with {}", output.status);
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&output.stdout).into_owned())
|
||||
}
|
||||
|
||||
fn lint(opts: &Opts, extra: &[String]) -> anyhow::Result<(bool, String)> {
|
||||
let output = Command::new(&opts.lint)
|
||||
.arg("--path")
|
||||
.arg(opts.out.join("questions"))
|
||||
.args(extra)
|
||||
.output()
|
||||
.with_context(|| format!("running {}", opts.lint))?;
|
||||
let mut report = String::from_utf8_lossy(&output.stdout).into_owned();
|
||||
report.push_str(&String::from_utf8_lossy(&output.stderr));
|
||||
Ok((output.status.success(), report))
|
||||
}
|
||||
|
||||
/// `<<<FILE path>>> ... <<<END>>>` blocks. Anything outside a block is
|
||||
/// ignored, so a model that adds a sentence does no harm.
|
||||
fn parse_files(text: &str) -> anyhow::Result<BTreeMap<String, String>> {
|
||||
// Small models mangle the markers: `<<<END>>` with a bracket short,
|
||||
// a stray `</think>`, the next `<<<FILE` glued onto the same line.
|
||||
// So markers are found anywhere in a line, an END needs only its
|
||||
// opening, and a new FILE closes whatever was open.
|
||||
let mut files = BTreeMap::new();
|
||||
let mut current: Option<(String, String)> = None;
|
||||
let close = |current: &mut Option<(String, String)>, files: &mut BTreeMap<String, String>| {
|
||||
if let Some((path, body)) = current.take() {
|
||||
files.insert(path, body);
|
||||
}
|
||||
};
|
||||
for line in text.lines() {
|
||||
let mut rest = line;
|
||||
loop {
|
||||
let (file_at, end_at) = (rest.find("<<<FILE "), rest.find("<<<END"));
|
||||
match (file_at, end_at) {
|
||||
(Some(f), e) if e.is_none_or(|e| f < e) => {
|
||||
close(&mut current, &mut files);
|
||||
let after = &rest[f + "<<<FILE ".len()..];
|
||||
let stop = after.find(">>>").unwrap_or(after.len());
|
||||
current = Some((after[..stop].trim().to_string(), String::new()));
|
||||
rest = after.get(stop + 3..).unwrap_or("");
|
||||
}
|
||||
(_, Some(e)) => {
|
||||
close(&mut current, &mut files);
|
||||
rest = &rest[e + "<<<END".len()..];
|
||||
}
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
// A line that carried a marker carries no content.
|
||||
if std::ptr::eq(rest, line) {
|
||||
if let Some((_, body)) = current.as_mut() {
|
||||
body.push_str(line);
|
||||
body.push('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
close(&mut current, &mut files);
|
||||
if files.is_empty() {
|
||||
bail!("the model returned no <<<FILE ...>>> blocks");
|
||||
}
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Model output is written to disk, so the path is checked, not
|
||||
/// trusted: repo-relative, yaml, and only where content lives.
|
||||
fn allowed(path: &str) -> bool {
|
||||
let p = Path::new(path);
|
||||
let plain = p.components().all(|c| matches!(c, std::path::Component::Normal(_)));
|
||||
plain
|
||||
&& p.extension().and_then(|e| e.to_str()) == Some("yaml")
|
||||
&& (matches!(path, "aggregates.yaml" | "site.yaml") || path.starts_with("questions/"))
|
||||
}
|
||||
|
||||
fn apply(out: &Path, files: BTreeMap<String, String>) -> anyhow::Result<()> {
|
||||
for (path, body) in files {
|
||||
if !allowed(&path) {
|
||||
eprintln!("ignored {path:?}: outside what the generator may write");
|
||||
continue;
|
||||
}
|
||||
let target = out.join(&path);
|
||||
if body.trim() == "DELETE" {
|
||||
std::fs::remove_file(&target).ok();
|
||||
continue;
|
||||
}
|
||||
if let Some(parent) = target.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
std::fs::write(target, body)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn content_files(out: &Path) -> anyhow::Result<Vec<PathBuf>> {
|
||||
fn walk(dir: &Path, found: &mut Vec<PathBuf>) -> anyhow::Result<()> {
|
||||
for entry in std::fs::read_dir(dir)? {
|
||||
let path = entry?.path();
|
||||
if path.is_dir() {
|
||||
walk(&path, found)?;
|
||||
} else if path.extension().and_then(|e| e.to_str()) == Some("yaml") {
|
||||
found.push(path);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
let mut found = Vec::new();
|
||||
for name in ["aggregates.yaml", "site.yaml"] {
|
||||
if out.join(name).exists() {
|
||||
found.push(out.join(name));
|
||||
}
|
||||
}
|
||||
walk(&out.join("questions"), &mut found)?;
|
||||
found.sort();
|
||||
Ok(found)
|
||||
}
|
||||
|
||||
fn snapshot(out: &Path) -> anyhow::Result<Vec<(PathBuf, String)>> {
|
||||
content_files(out)?
|
||||
.into_iter()
|
||||
.map(|p| Ok((p.clone(), std::fs::read_to_string(p)?)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn restore(out: &Path, before: Vec<(PathBuf, String)>) -> anyhow::Result<()> {
|
||||
for path in content_files(out)? {
|
||||
std::fs::remove_file(path)?;
|
||||
}
|
||||
for (path, body) in before {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
std::fs::write(path, body)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn dump(out: &Path) -> anyhow::Result<String> {
|
||||
let mut text = String::new();
|
||||
for path in content_files(out)? {
|
||||
let rel = path.strip_prefix(out).unwrap_or(&path).display().to_string();
|
||||
text.push_str(&format!("<<<FILE {rel}>>>\n{}<<<END>>>\n\n", std::fs::read_to_string(&path)?));
|
||||
}
|
||||
Ok(text)
|
||||
}
|
||||
|
||||
/// The answering side never sees `expect` or which need a persona has.
|
||||
///
|
||||
/// One call per task, so no visitor sees another visitor's situation
|
||||
/// (a batch of eight personas over four headings is nearly a matching
|
||||
/// puzzle, and a model solves it as one). The options are shuffled per
|
||||
/// task, so a heading's place on the page cannot carry the answer, and
|
||||
/// the call is seeded from the task id, so a rerun on the same site is
|
||||
/// the same experiment.
|
||||
///
|
||||
/// With a content repo to read, the visitor sees each option as the
|
||||
/// card portal renders: button, encouragement, sections with icon and
|
||||
/// guidance, fields with placeholders. Headings-only tasks (`--skim`,
|
||||
/// where the task's text is the heading itself) stay headings only.
|
||||
pub(crate) fn answer_blind(cmd: &str, tasks: &str, repo: Option<&Path>, seed_offset: u64) -> anyhow::Result<String> {
|
||||
let mut answers = String::new();
|
||||
for line in tasks.lines().filter(|l| !l.trim().is_empty()) {
|
||||
let task: serde_json::Value = serde_json::from_str(line)?;
|
||||
let id = task["id"].as_str().unwrap_or_default();
|
||||
let body = task["state"]["body"].as_str().unwrap_or_default();
|
||||
let pick = &task["questions"]["pick"];
|
||||
let instructions = pick["instructions"].as_str().unwrap_or_default();
|
||||
let mut options: Vec<(&str, &str)> = pick["criteria"]
|
||||
.as_object()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|(k, v)| (k.as_str(), v.as_str().unwrap_or_default()))
|
||||
.collect();
|
||||
let skim = options.iter().all(|(name, text)| text.is_empty() || text == name);
|
||||
let page = if skim { None } else { repo.and_then(|r| rate::load_page(r, task["page"].as_str().unwrap_or("/"))) };
|
||||
let seed = fnv(id).wrapping_add(seed_offset.wrapping_mul(0x9E37_79B9_7F4A_7C15) as _);
|
||||
shuffle(&mut options, seed);
|
||||
// Options are numbered and the answer is the number: with a
|
||||
// whole card in view, a small model asked to copy a heading
|
||||
// copied the page's description instead (measured: six of
|
||||
// fourteen misroutes on one run were that), and no heading
|
||||
// matched. A number cannot be a description.
|
||||
let listed: Vec<String> = options
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (name, text))| {
|
||||
let card = page
|
||||
.as_ref()
|
||||
.and_then(|p| p["alternatives"].as_array())
|
||||
.and_then(|alts| alts.iter().find(|a| a["name"] == *name))
|
||||
.map(|a| card::alternative(a, true));
|
||||
let body = match card {
|
||||
Some(card) => card.trim_end().trim_start_matches("- ").to_string(),
|
||||
None if skim => name.to_string(),
|
||||
None => format!("{name}: {text}"),
|
||||
};
|
||||
format!("{}. {body}", i + 1)
|
||||
})
|
||||
.collect();
|
||||
let mut head = page.as_ref().map(card::page_head).unwrap_or_default();
|
||||
// What the page says before it offers anything (the opener, the
|
||||
// look around): read, not picked.
|
||||
if let Some(context) = task["state"]["page"].as_str().filter(|c| !c.trim().is_empty()) {
|
||||
head.push_str(&format!("The page first says:\n{context}\n\n"));
|
||||
}
|
||||
let prompt = format!(
|
||||
"You are this person: {body}\n\n{instructions}\n{head}The page offers these options:\n\n{}\n\nWhich one do you pick? Judge only from your own situation and the wording in front of you. Give your honest confidence between 0 and 1; if two options fit you about equally, confidence is low.\nReply as JSON: {{\"choice\": <the option's number>, \"confidence\": 0.00}}",
|
||||
listed.join("\n\n")
|
||||
);
|
||||
let reply = llm_env(cmd, &prompt, &[("FORMAT", "json"), ("SEED", &seed.to_string()), ("TEMPERATURE", "0.2")])?;
|
||||
let parsed = simulate::first_object(&reply).unwrap_or(serde_json::Value::Null);
|
||||
let names: Vec<&str> = options.iter().map(|(n, _)| *n).collect();
|
||||
let raw = match &parsed["choice"] {
|
||||
serde_json::Value::Number(n) => n.to_string(),
|
||||
other => other.as_str().unwrap_or_default().to_string(),
|
||||
};
|
||||
// A number, or a heading for a model that wrote one anyway.
|
||||
let choice = match raw.trim().trim_end_matches('.').parse::<usize>() {
|
||||
Ok(n) if (1..=names.len()).contains(&n) => names[n - 1].to_string(),
|
||||
_ => simulate::resolve_choice(&names, &raw),
|
||||
};
|
||||
let confidence = parsed["confidence"].as_f64().unwrap_or(0.0);
|
||||
answers.push_str(&serde_json::to_string(&serde_json::json!({ "id": id, "choice": choice, "confidence": confidence }))?);
|
||||
answers.push('\n');
|
||||
}
|
||||
Ok(answers)
|
||||
}
|
||||
|
||||
/// FNV-1a over the task id: a seed that is the same on every run.
|
||||
fn fnv(text: &str) -> u32 {
|
||||
text.bytes().fold(0x811c_9dc5u32, |h, b| (h ^ u32::from(b)).wrapping_mul(0x0100_0193))
|
||||
}
|
||||
|
||||
/// Fisher-Yates on a small xorshift, so the order is fixed by the seed.
|
||||
fn shuffle<T>(items: &mut [T], seed: u32) {
|
||||
let mut x = seed.max(1);
|
||||
for i in (1..items.len()).rev() {
|
||||
x ^= x << 13;
|
||||
x ^= x >> 17;
|
||||
x ^= x << 5;
|
||||
items.swap(i, (x as usize) % (i + 1));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn files_are_parsed_from_blocks_and_prose_is_ignored() {
|
||||
let files = parse_files("Sure!\n<<<FILE site.yaml>>>\ntitle: A\n<<<END>>>\nnoise\n<<<FILE questions/index.yaml>>>\nname: Q?\n<<<END>>>\n").unwrap();
|
||||
assert_eq!(files["site.yaml"], "title: A\n");
|
||||
assert_eq!(files.len(), 2);
|
||||
assert!(parse_files("no blocks here").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mangled_markers_from_a_small_model_still_separate_the_files() {
|
||||
// Verbatim shape of a nemotron reply: a short END, a stray
|
||||
// think tag, the next FILE glued on, and no END at the very end.
|
||||
let files = parse_files(
|
||||
"<<<FILE questions/desk/index.yaml>>>\nname: Desk?\n<<<END>></think><<<FILE questions/index.yaml>>>\nname: Hva trenger du?\n<<<FILE site.yaml>>>\ntitle: A\n",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(files["questions/desk/index.yaml"], "name: Desk?\n");
|
||||
assert_eq!(files["questions/index.yaml"], "name: Hva trenger du?\n");
|
||||
assert_eq!(files["site.yaml"], "title: A\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_task_is_shuffled_the_same_way_every_time_and_differently_from_the_next() {
|
||||
let mut a = ["w", "x", "y", "z"];
|
||||
let mut b = a;
|
||||
shuffle(&mut a, fnv("scaffolder@/"));
|
||||
shuffle(&mut b, fnv("scaffolder@/"));
|
||||
assert_eq!(a, b);
|
||||
let mut c = ["w", "x", "y", "z"];
|
||||
shuffle(&mut c, fnv("apprentice@/"));
|
||||
assert_ne!(a, c);
|
||||
let mut sorted = a;
|
||||
sorted.sort();
|
||||
assert_eq!(sorted, ["w", "x", "y", "z"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_content_paths_may_be_written() {
|
||||
assert!(allowed("site.yaml"));
|
||||
assert!(allowed("aggregates.yaml"));
|
||||
assert!(allowed("questions/desk/_section.yaml"));
|
||||
// The spec is frozen after phase one.
|
||||
assert!(!allowed("needs.yaml"));
|
||||
assert!(!allowed("../outside.yaml"));
|
||||
assert!(!allowed("/etc/passwd.yaml"));
|
||||
assert!(!allowed("questions/run.sh"));
|
||||
assert!(!allowed(".gitea/workflows/deploy.yaml"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user