build: a reply in the wrong language keeps the words it had; the vel's mails
A model asked for Norwegian sometimes answers in English. The commonest function words of each language are counted across a reply, and one that reads as the other language is refused with a line saying so. Tomter Vel's six mails, worded by Sonnet in Norwegian, are round 2 of runs/tomtervel-opener-2026-09-23 and live. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
72a081ed59
commit
8c7fb870bf
+57
-2
@@ -934,6 +934,43 @@ pub fn merge_words(fallback: &Value, given: &Value) -> Value {
|
||||
|
||||
const SLOT: &str = "<WRITE";
|
||||
|
||||
/// A model asked for Norwegian sometimes answers in English, and the
|
||||
/// reverse; the words then look fine and read wrong. Counts the
|
||||
/// commonest function words of each language across every string of
|
||||
/// a reply and names the language that won when it is not the plan's.
|
||||
/// Only the two languages the plans come in; any other language is
|
||||
/// left unchecked.
|
||||
fn wrong_language(lang: &str, reply: &Value) -> Option<&'static str> {
|
||||
const NORWEGIAN: &[&str] = &["og", "er", "du", "det", "ikke", "til", "som", "med", "på", "har", "vi", "deg", "en", "et"];
|
||||
const ENGLISH: &[&str] = &["the", "and", "you", "is", "are", "we", "to", "of", "your", "it", "in", "with", "for", "a"];
|
||||
fn collect(v: &Value, out: &mut String) {
|
||||
match v {
|
||||
Value::String(s) => {
|
||||
out.push(' ');
|
||||
out.push_str(s);
|
||||
}
|
||||
Value::Object(m) => m.values().for_each(|v| collect(v, out)),
|
||||
Value::Array(a) => a.iter().for_each(|v| collect(v, out)),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let mut text = String::new();
|
||||
collect(reply, &mut text);
|
||||
let words: Vec<String> = text.split(|c: char| !c.is_alphabetic()).filter(|w| !w.is_empty()).map(|w| w.to_lowercase()).collect();
|
||||
if words.len() < 8 {
|
||||
return None;
|
||||
}
|
||||
let no = words.iter().filter(|w| NORWEGIAN.contains(&w.as_str())).count();
|
||||
let en = words.iter().filter(|w| ENGLISH.contains(&w.as_str())).count();
|
||||
let plan_is_norwegian = lang.starts_with("no") || lang.starts_with("nb") || lang.starts_with("nn");
|
||||
let plan_is_english = lang.starts_with("en");
|
||||
match (plan_is_norwegian, plan_is_english) {
|
||||
(true, _) if en > no * 2 && en >= 4 => Some("English"),
|
||||
(_, true) if no > en * 2 && no >= 4 => Some("Norwegian"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// A mail's link to the submitter's own move is structure, not
|
||||
/// wording: a model told to keep it still drops it (Sonnet replaced
|
||||
/// the whole line with a bare `{chain}`). Wherever the fallback body
|
||||
@@ -1514,8 +1551,13 @@ pub fn run(argv: &[String]) -> anyhow::Result<()> {
|
||||
if path == "aggregates.yaml" {
|
||||
keep_links(fallback, &mut merged);
|
||||
}
|
||||
words.insert(path.clone(), merged);
|
||||
eprintln!("words: {path}");
|
||||
match wrong_language(&plan.language, &given) {
|
||||
Some(found) => eprintln!("words: {path} - the reply reads as {found} for a {} plan, keeping the words it had", plan.language),
|
||||
None => {
|
||||
words.insert(path.clone(), merged);
|
||||
eprintln!("words: {path}");
|
||||
}
|
||||
}
|
||||
}
|
||||
None => eprintln!("words: {path} - no usable reply, keeping the fallback"),
|
||||
}
|
||||
@@ -1592,6 +1634,19 @@ mod tests {
|
||||
assert!(serde_yaml::from_str::<Plan>(&no_from).unwrap().validate().unwrap_err().to_string().contains("mail: { from"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_reply_in_the_wrong_language_is_named() {
|
||||
let english = json!({ "a": { "body": "We have received what you sent and we will read it in full before we reply to you." } });
|
||||
let norwegian = json!({ "a": { "body": "Vi har mottatt det du sendte og leser det i sin helhet før vi svarer deg på det." } });
|
||||
assert_eq!(wrong_language("no", &english), Some("English"));
|
||||
assert_eq!(wrong_language("no", &norwegian), None);
|
||||
assert_eq!(wrong_language("en", &norwegian), Some("Norwegian"));
|
||||
assert_eq!(wrong_language("en", &english), None);
|
||||
// Too short to tell, or a language nobody checks: left alone.
|
||||
assert_eq!(wrong_language("no", &json!({ "a": "Send" })), None);
|
||||
assert_eq!(wrong_language("de", &english), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_models_mail_keeps_the_submitters_link() {
|
||||
let fallback = json!({ "leaks": { "open": { "subject": "s", "body": "Hello,\n\nGot it.\n\nWhat you can do next: {site}/decide/leak?chain={chain}&email={email}\n\nI at i@b.example" } } });
|
||||
|
||||
Reference in New Issue
Block a user