uhhm/portal
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 89 Wiki Activity

generate: a Norwegian run on a local model, and what it took to read it

Browse Source 
Every prompt and reply is now kept under <out>/llm/. An empty reply is
a retry, not a fatal error; a spec that never parses stops the run
before the structure phase; --visitor-llm lets a different model play
the visitor. The file-block parser finds markers anywhere in a line,
accepts a short END and lets a new FILE close an open one, after a
reply glued two files together with `<<<END>></think>`. The reference
gains a language section: copy in the intake's language, identifiers
in ASCII.

runs/rehab-entreprenor-no-2026-09-21: nemotron-3.5-lightning:30b-a3b
did not produce a loadable site. Written up with the reason and the
way forward: structure from needs.yaml deterministically, words from
the model.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L4jrCgLiKKHAEFuZUJjckH
This commit is contained in:
Bendik Aagaard Lynghaug
2026-09-21 20:21:46 +02:00
co-authored by Claude Fable 5.1
parent b9f2acb7b8
commit b53e7e6aca
17 changed files with 2123 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.rs
+83 -12
View File
@@ -34,6 +34,9 @@ struct Opts {
llm: String,
rounds: usize,
wording_rounds: usize,
/// The model that plays the visitor. A generator grading its own
/// wording agrees with itself, so this should be a different one.
visitor: Option<String>,
}
fn main() -> anyhow::Result<()> {
@@ -64,6 +67,8 @@ fn main() -> anyhow::Result<()> {
let intake = std::fs::read_to_string(&opts.intake)
.with_context(|| format!("reading {}", opts.intake.display()))?;
std::fs::create_dir_all(opts.out.join("questions"))?;
std::fs::create_dir_all(opts.out.join("llm"))?;
LLM_LOG.set(opts.out.join("llm")).ok();
let mut log = Vec::new();
// Phase 1: the spec.
@@ -72,7 +77,11 @@ fn main() -> anyhow::Result<()> {
let prompt = format!(
"{REFERENCE}\n\n# Task\n\nA business filled in this intake. Write ONLY `needs.yaml` for it: every need the intake implies, for the public and for each role inside the business, with states taken from how the business says things progress, and personas.\n\n## Intake\n\n{intake}\n{feedback}\n{FORMAT}"
);
let files = parse_files(&llm(&opts.llm, &prompt)?)?;
let Ok(files) = parse_files(&llm(&opts.llm, &prompt)?) else {
eprintln!("needs round {round}: the reply had no file blocks, asking again");
feedback = "\n## Your last attempt\n\nIt contained no <<<FILE ...>>> blocks at all. Return needs.yaml in exactly that format.\n".to_string();
continue;
};
let Some(needs) = files.get("needs.yaml") else {
feedback = "\n## Your last attempt\n\nIt contained no needs.yaml.\n".to_string();
continue;
@@ -89,8 +98,12 @@ fn main() -> anyhow::Result<()> {
}
eprintln!("needs round {round}: needs.yaml parses, frozen");
log.push(format!("needs: parsed in round {round}"));
feedback.clear();
break;
}
if !feedback.is_empty() {
bail!("needs.yaml never parsed in {} round(s); the replies are in {}/llm/", opts.rounds, opts.out.display());
}
let needs = std::fs::read_to_string(opts.out.join("needs.yaml")).context("no needs.yaml was produced")?;
// Phase 2: structure. The page-writing model sees the needs and
@@ -116,7 +129,10 @@ fn main() -> anyhow::Result<()> {
dump(&opts.out)?
)
};
apply(&opts.out, parse_files(&llm(&opts.llm, &prompt)?)?)?;
match parse_files(&llm(&opts.llm, &prompt)?) {
Ok(files) => apply(&opts.out, files)?,
Err(_) => eprintln!("structure round {round}: the reply had no file blocks"),
}
let (ok, r) = lint(&opts, &[])?;
report = r;
let fails = report.lines().filter(|l| l.starts_with("FAIL")).count();
@@ -143,7 +159,7 @@ fn main() -> anyhow::Result<()> {
args.push("--skim".to_string());
}
lint(&opts, &args)?;
let answers = answer_blind(&opts.llm, &std::fs::read_to_string(&tasks_path)?)?;
let answers = answer_blind(opts.visitor.as_deref().unwrap_or(&opts.llm), &std::fs::read_to_string(&tasks_path)?)?;
let answers_path = opts.out.join(if skim { "answers-skim.jsonl" } else { "answers.jsonl" });
std::fs::write(&answers_path, answers)?;
let mut args = vec!["--needs-score".to_string(), answers_path.display().to_string()];
@@ -198,6 +214,7 @@ fn parse_args() -> anyhow::Result<Opts> {
llm: "claude -p".to_string(),
rounds: 4,
wording_rounds: 2,
visitor: None,
};
while let Some(arg) = args.next() {
let mut value = || args.next().ok_or_else(|| anyhow!("{arg} needs a value"));
@@ -208,6 +225,7 @@ fn parse_args() -> anyhow::Result<Opts> {
"--llm" => opts.llm = value()?,
"--rounds" => opts.rounds = value()?.parse()?,
"--wording-rounds" => opts.wording_rounds = value()?.parse()?,
"--visitor-llm" => opts.visitor = Some(value()?),
other => bail!("unknown argument {other}"),
}
}
@@ -218,8 +236,23 @@ fn parse_args() -> anyhow::Result<Opts> {
}
/// Prompt on stdin, text on stdout.
/// Where `generate` keeps every prompt and reply (`<out>/llm/`). A
/// model that returns nothing usable has to be read, not guessed at.
static LLM_LOG: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
static LLM_CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
fn llm(cmd: &str, prompt: &str) -> anyhow::Result<String> {
llm_with(cmd, prompt, &[])
let reply = llm_with(cmd, prompt, &[]);
if let Some(dir) = LLM_LOG.get() {
let n = LLM_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
std::fs::write(dir.join(format!("{n:02}-prompt.md")), prompt).ok();
let text = match &reply {
Ok(text) => text.clone(),
Err(e) => format!("ERROR: {e}"),
};
std::fs::write(dir.join(format!("{n:02}-reply.md")), text).ok();
}
reply
}
/// As `llm`, with hints for the backend in its environment (`SEED`,
@@ -262,20 +295,45 @@ fn lint(opts: &Opts, extra: &[String]) -> anyhow::Result<(bool, String)> {
/// `<<<FILE path>>> ... <<<END>>>` blocks. Anything outside a block is
/// ignored, so a model that adds a sentence does no harm.
fn parse_files(text: &str) -> anyhow::Result<BTreeMap<String, String>> {
// Small models mangle the markers: `<<<END>>` with a bracket short,
// a stray `</think>`, the next `<<<FILE` glued onto the same line.
// So markers are found anywhere in a line, an END needs only its
// opening, and a new FILE closes whatever was open.
let mut files = BTreeMap::new();
let mut current: Option<(String, String)> = None;
let mut close = |current: &mut Option<(String, String)>, files: &mut BTreeMap<String, String>| {
if let Some((path, body)) = current.take() {
files.insert(path, body);
}
};
for line in text.lines() {
if let Some(path) = line.trim().strip_prefix("<<<FILE ").and_then(|r| r.strip_suffix(">>>")) {
current = Some((path.trim().to_string(), String::new()));
} else if line.trim() == "<<<END>>>" {
if let Some((path, body)) = current.take() {
files.insert(path, body);
let mut rest = line;
loop {
let (file_at, end_at) = (rest.find("<<<FILE "), rest.find("<<<END"));
match (file_at, end_at) {
(Some(f), e) if e.is_none_or(|e| f < e) => {
close(&mut current, &mut files);
let after = &rest[f + "<<<FILE ".len()..];
let stop = after.find(">>>").unwrap_or(after.len());
current = Some((after[..stop].trim().to_string(), String::new()));
rest = after.get(stop + 3..).unwrap_or("");
}
(_, Some(e)) => {
close(&mut current, &mut files);
rest = &rest[e + "<<<END".len()..];
}
_ => break,
}
}
// A line that carried a marker carries no content.
if std::ptr::eq(rest, line) {
if let Some((_, body)) = current.as_mut() {
body.push_str(line);
body.push('\n');
}
} else if let Some((_, body)) = current.as_mut() {
body.push_str(line);
body.push('\n');
}
}
close(&mut current, &mut files);
if files.is_empty() {
bail!("the model returned no <<<FILE ...>>> blocks");
}
@@ -403,6 +461,19 @@ mod tests {
assert!(parse_files("no blocks here").is_err());
}
#[test]
fn mangled_markers_from_a_small_model_still_separate_the_files() {
// Verbatim shape of a nemotron reply: a short END, a stray
// think tag, the next FILE glued on, and no END at the very end.
let files = parse_files(
"<<<FILE questions/desk/index.yaml>>>\nname: Desk?\n<<<END>></think><<<FILE questions/index.yaml>>>\nname: Hva trenger du?\n<<<FILE site.yaml>>>\ntitle: A\n",
)
.unwrap();
assert_eq!(files["questions/desk/index.yaml"], "name: Desk?\n");
assert_eq!(files["questions/index.yaml"], "name: Hva trenger du?\n");
assert_eq!(files["site.yaml"], "title: A\n");
}
#[test]
fn only_content_paths_may_be_written() {
assert!(allowed("site.yaml"));