From cc50249d7fded21dfcf077dc01a5005b91b20043 Mon Sep 17 00:00:00 2001 From: Bendik Aagaard Lynghaug Date: Wed, 23 Sep 2026 22:46:25 +0200 Subject: [PATCH] answer --seed, and runs/spread.sh: a score comes with its spread Every score so far was one run of a small visitor. `answer --seed n` answers the same tasks with other shuffles and sampling, and runs/spread.sh prints a run's score per seed, so a change can be told from the visitor's mood. Co-Authored-By: Claude Fable 5.1 --- runs/spread.sh | 19 +++++++++++++++++++ src/main.rs | 14 +++++++++----- src/score.rs | 2 +- 3 files changed, 29 insertions(+), 6 deletions(-) create mode 100755 runs/spread.sh diff --git a/runs/spread.sh b/runs/spread.sh new file mode 100755 index 0000000..e7e485b --- /dev/null +++ b/runs/spread.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# The noise floor: answer one run's tasks with several seeds and print +# each seed's score, so a change can be told from the visitor's mood. +# runs/spread.sh runs/ [seeds...] (default seeds: 1 2 3) +set -eu +run=${1:?usage: spread.sh runs/ [seeds...]}; shift +seeds=${*:-1 2 3} +IRIS=/var/local/cargo-target/release/iris +T=/var/local/cargo-target/release/portal-trainer +LLM="MODEL=qwen3:8b $(dirname "$0")/../engines/ollama.sh" +cd "$run" +for mode in full skim; do + skim=""; [ "$mode" = skim ] && skim="--skim" + for s in $seeds; do + $T answer --tasks tasks-$mode.jsonl --repo site --llm "$LLM" --seed "$s" > answers-$mode-seed$s.jsonl + printf "%s seed %s: " "$mode" "$s" + $IRIS check --path site/questions --needs-tasks tasks-$mode.jsonl $skim --needs-score answers-$mode-seed$s.jsonl 2>&1 | grep -E "^wording:" + done +done diff --git a/src/main.rs b/src/main.rs index b023e6d..6a6f56a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,10 +65,14 @@ fn main() -> anyhow::Result<()> { .and_then(|i| argv.get(i + 1)) .cloned() }; - let tasks = value("--tasks").ok_or_else(|| anyhow!("usage: portal-trainer answer --tasks [--repo ] [--llm 'cmd']"))?; + let tasks = value("--tasks").ok_or_else(|| anyhow!("usage: portal-trainer answer --tasks [--repo ] [--llm 'cmd'] [--seed ]"))?; let cmd = value("--llm").unwrap_or_else(|| "claude -p".to_string()); let repo = value("--repo").map(PathBuf::from); - print!("{}", answer_blind(&cmd, &std::fs::read_to_string(&tasks).with_context(|| format!("reading {tasks}"))?, repo.as_deref())?); + // --seed: the same tasks answered again with other shuffles and + // sampling, so a score comes with its spread. 0 is the run + // every earlier score used. + let seed: u64 = value("--seed").and_then(|s| s.parse().ok()).unwrap_or(0); + print!("{}", answer_blind(&cmd, &std::fs::read_to_string(&tasks).with_context(|| format!("reading {tasks}"))?, repo.as_deref(), seed)?); return Ok(()); } let opts = parse_args()?; @@ -167,7 +171,7 @@ fn main() -> anyhow::Result<()> { args.push("--skim".to_string()); } lint(&opts, &args)?; - let answers = answer_blind(opts.visitor.as_deref().unwrap_or(&opts.llm), &std::fs::read_to_string(&tasks_path)?, Some(&opts.out))?; + let answers = answer_blind(opts.visitor.as_deref().unwrap_or(&opts.llm), &std::fs::read_to_string(&tasks_path)?, Some(&opts.out), 0)?; let answers_path = opts.out.join(if skim { "answers-skim.jsonl" } else { "answers.jsonl" }); std::fs::write(&answers_path, answers)?; let mut args = vec!["--needs-score".to_string(), answers_path.display().to_string()]; @@ -447,7 +451,7 @@ fn dump(out: &Path) -> anyhow::Result { /// card portal renders: button, encouragement, sections with icon and /// guidance, fields with placeholders. Headings-only tasks (`--skim`, /// where the task's text is the heading itself) stay headings only. -pub(crate) fn answer_blind(cmd: &str, tasks: &str, repo: Option<&Path>) -> anyhow::Result { +pub(crate) fn answer_blind(cmd: &str, tasks: &str, repo: Option<&Path>, seed_offset: u64) -> anyhow::Result { let mut answers = String::new(); for line in tasks.lines().filter(|l| !l.trim().is_empty()) { let task: serde_json::Value = serde_json::from_str(line)?; @@ -463,7 +467,7 @@ pub(crate) fn answer_blind(cmd: &str, tasks: &str, repo: Option<&Path>) -> anyho .collect(); let skim = options.iter().all(|(name, text)| text.is_empty() || text == name); let page = if skim { None } else { repo.and_then(|r| rate::load_page(r, task["page"].as_str().unwrap_or("/"))) }; - let seed = fnv(id); + let seed = fnv(id).wrapping_add(seed_offset.wrapping_mul(0x9E37_79B9_7F4A_7C15) as _); shuffle(&mut options, seed); // Options are numbered and the answer is the number: with a // whole card in view, a small model asked to copy a heading diff --git a/src/score.rs b/src/score.rs index e5f801b..3a7d690 100644 --- a/src/score.rs +++ b/src/score.rs @@ -34,7 +34,7 @@ pub fn run(argv: &[String]) -> anyhow::Result<()> { let mut summary = serde_json::Map::new(); for (mode, tasks, skim) in [("full", &tasks_full, false), ("skim", &tasks_skim, true)] { eprintln!("score: the visitor answers {mode}"); - let answers = crate::answer_blind(&cmd, &std::fs::read_to_string(tasks)?, Some(&site))?; + let answers = crate::answer_blind(&cmd, &std::fs::read_to_string(tasks)?, Some(&site), 0)?; let answers_path = out.join(format!("answers-{mode}.jsonl")); std::fs::write(&answers_path, &answers)?; let (tasks_arg, answers_arg) = (s(tasks), s(&answers_path));