Storylines: the same site told several ways, one per visitor, counted

Any text a person reads may be a list, one entry per storyline, every
list the same length. A visitor is given one storyline for the whole
site and reads it on every page and in every mail about their case.
Records and events carry the storyline; GET /report gives counts per
bucket, storyline and state, and nothing else.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GT37Z1Xtfd9pUuQtMTg6Yt

src/content/gitea.rs

@@ -9,20 +9,27 @@ use super::*;
 /// config; a file that exists but doesn't parse or validate is a real
 /// error, surfaced at boot rather than papered over.
 #[cfg(feature = "ssr")]
-pub async fn load_site_from_gitea(repo_url: &str, branch: &str) -> anyhow::Result<SiteConfig> {
+pub async fn load_site_from_gitea(repo_url: &str, branch: &str) -> anyhow::Result<Storylines<SiteConfig>> {
     let (owner, repo) = parse_owner_repo(repo_url)?;
     let api_base = gitea_api_base(repo_url)?;
     let raw = match fetch_gitea_file(&api_base, &owner, &repo, branch, "site.yaml").await {
         Ok(raw) => raw,
         Err(e) => {
             tracing::info!("no site.yaml in content repo ({e}), using default branding");
-            return Ok(SiteConfig::default());
+            return Ok(Storylines::one(SiteConfig::default()));
         }
     };
-    let site: SiteConfig =
-        serde_yaml::from_str(&raw).map_err(|e| anyhow::anyhow!("parsing site.yaml: {e}"))?;
-    site.validate()?;
-    Ok(site)
+    // The invite mail's words may be several (see `variants.rs`).
+    let files = vec![("site.yaml".to_string(), raw)];
+    let arity = super::variants::arity(&files)?;
+    let mut each = Vec::with_capacity(arity);
+    for n in 0..arity {
+        let site: SiteConfig =
+            serde_yaml::from_str(&super::variants::storyline(&files, n)[0].1).map_err(|e| anyhow::anyhow!("parsing site.yaml: {e}"))?;
+        site.validate()?;
+        each.push(site);
+    }
+    Ok(Storylines { each })
 }
 
 /// Extracts `scheme://host` from a repo's normal browser URL - the
@@ -107,11 +114,18 @@ pub(crate) async fn fetch_gitea_file(
 pub async fn load_aggregates_from_gitea(
     repo_url: &str,
     branch: &str,
-) -> anyhow::Result<std::collections::HashMap<String, crate::aggregates::AggregateSchema>> {
+) -> anyhow::Result<Storylines<std::collections::HashMap<String, crate::aggregates::AggregateSchema>>> {
     let (owner, repo) = parse_owner_repo(repo_url)?;
     let api_base = gitea_api_base(repo_url)?;
     let raw = fetch_gitea_file(&api_base, &owner, &repo, branch, "aggregates.yaml").await?;
-    crate::aggregates::parse_aggregates_yaml(&raw)
+    // A mail's subject and body may be several (see `variants.rs`);
+    // the runtime's own buckets are added to every storyline.
+    let files = vec![("aggregates.yaml".to_string(), raw)];
+    let arity = super::variants::arity(&files)?;
+    let each = (0..arity)
+        .map(|n| crate::aggregates::parse_aggregates_yaml(&super::variants::storyline(&files, n)[0].1).map(with_builtin_aggregates))
+        .collect::<anyhow::Result<Vec<_>>>()?;
+    Ok(Storylines { each })
 }
 
 #[cfg(feature = "ssr")]
@@ -119,7 +133,7 @@ pub async fn load_questions_from_gitea(
     repo_url: &str,
     branch: &str,
     subdir: &str,
-) -> anyhow::Result<std::collections::HashMap<String, Question>> {
+) -> anyhow::Result<Storylines<std::collections::HashMap<String, Question>>> {
     let (owner, repo) = parse_owner_repo(repo_url)?;
     let api_base = gitea_api_base(repo_url)?;
 
@@ -161,7 +175,20 @@ pub async fn load_questions_from_gitea(
             .map_err(|e| anyhow::anyhow!("fetching {path}: {e}"))?;
         files.push((path[prefix.len()..].to_string(), raw));
     }
-    build_questions(&files)
+    build_storylines(&files)
+}
+
+/// Every storyline's arity is the site's or one: a mail told in two
+/// voices on a site whose pages are told in three has a hole in it.
+#[cfg(feature = "ssr")]
+pub fn same_arity(questions: usize, aggregates: usize, site: usize) -> anyhow::Result<usize> {
+    let arity = questions.max(aggregates).max(site);
+    for (what, n) in [("questions", questions), ("aggregates.yaml", aggregates), ("site.yaml", site)] {
+        if n != 1 && n != arity {
+            anyhow::bail!("{what} tells {n} storyline(s), but the site tells {arity}; every list of texts has the same length");
+        }
+    }
+    Ok(arity)
 }
 
 /// Published by the content repo's own CI (after it lints a push) to
@@ -192,11 +219,11 @@ pub async fn watch_for_reload(
     repo_url: String,
     branch: String,
     subdir: String,
-    questions: std::sync::Arc<arc_swap::ArcSwap<std::collections::HashMap<String, Question>>>,
+    questions: std::sync::Arc<arc_swap::ArcSwap<Storylines<std::collections::HashMap<String, Question>>>>,
     aggregates: std::sync::Arc<
-        arc_swap::ArcSwap<std::collections::HashMap<String, crate::aggregates::AggregateSchema>>,
+        arc_swap::ArcSwap<Storylines<std::collections::HashMap<String, crate::aggregates::AggregateSchema>>>,
     >,
-    site: std::sync::Arc<arc_swap::ArcSwap<SiteConfig>>,
+    site: std::sync::Arc<arc_swap::ArcSwap<Storylines<SiteConfig>>>,
     access: std::sync::Arc<arc_swap::ArcSwap<crate::access::Policy>>,
 ) {
     let mut sub = match nats.subscribe(CONTENT_RELOAD_SUBJECT).await {
@@ -209,7 +236,7 @@ pub async fn watch_for_reload(
     use futures::StreamExt;
     while sub.next().await.is_some() {
         let loaded_aggregates = match load_aggregates_from_gitea(&repo_url, &branch).await {
-            Ok(loaded) => with_builtin_aggregates(loaded),
+            Ok(loaded) => loaded,
             Err(e) => {
                 tracing::error!(error = %e, "aggregates.yaml reload failed, keeping last-good content");
                 continue;
@@ -228,7 +255,7 @@ pub async fn watch_for_reload(
         };
         match load_questions_from_gitea(&repo_url, &branch, &subdir).await {
             Ok(loaded) => {
-                if let Err(e) = validate_questions(&loaded, &loaded_aggregates) {
+                if let Err(e) = validate_questions(&loaded, &loaded_aggregates).and_then(|_| same_arity(loaded.arity(), loaded_aggregates.arity(), loaded_site.arity())) {
                     tracing::error!(error = %e, "content reload failed validation, keeping last-good content");
                     continue;
                 }

src/content/mod.rs

@@ -13,12 +13,16 @@ use serde::{Deserialize, Serialize};
 // Loading is server-only; the rest is shared with the browser build.
 #[cfg(feature = "ssr")]
 mod gitea;
+#[cfg(feature = "ssr")]
+mod variants;
 mod handlers;
 mod markdown;
 mod validate;
 
 #[cfg(feature = "ssr")]
 pub use gitea::*;
+#[cfg(feature = "ssr")]
+pub use variants::*;
 pub use handlers::*;
 pub use markdown::*;
 pub use validate::*;
@@ -1073,6 +1077,25 @@ pub fn resolve_question(
 #[cfg(feature = "ssr")]
 pub fn build_questions(
     files: &[(String, String)],
+) -> anyhow::Result<std::collections::HashMap<String, Question>> {
+    // The first storyline, once every list of texts has been held to
+    // the site's arity (see `variants.rs`); the machinery reads this
+    // one, and `build_storylines` builds the rest for the visitors.
+    variants::arity(files)?;
+    build_one_storyline(&variants::storyline(files, 0))
+}
+
+/// Every storyline of the content, each a full map of questions.
+#[cfg(feature = "ssr")]
+pub fn build_storylines(files: &[(String, String)]) -> anyhow::Result<variants::Storylines<std::collections::HashMap<String, Question>>> {
+    let arity = variants::arity(files)?;
+    let each = (0..arity).map(|n| build_one_storyline(&variants::storyline(files, n))).collect::<anyhow::Result<Vec<_>>>()?;
+    Ok(variants::Storylines { each })
+}
+
+#[cfg(feature = "ssr")]
+fn build_one_storyline(
+    files: &[(String, String)],
 ) -> anyhow::Result<std::collections::HashMap<String, Question>> {
     // Sections first: (directory prefix, config), shallowest first so a
     // later (deeper) section overrides an outer one field-by-field.

src/content/variants.rs

@@ -0,0 +1,224 @@
+//! Storylines: a text may be several, and `[n]` runs through the site.
+//!
+//! Anywhere content says something to a person - a page's `name` and
+//! `description`, an alternative's, a feature's, a `label`, a
+//! `placeholder`, a mail's `subject` and `body`, a list's `empty` line,
+//! the `consequence` buttons and `encouragements` - it may give a list
+//! instead of one string:
+//!
+//! ```yaml
+//! description:
+//!   - Dugnad, 17. mai og komiteene trenger flere hender. Si hva du kan.
+//!   - Naboene gjør det meste selv. Her melder du deg til det du har lyst på.
+//! ```
+//!
+//! Every such list has the same length, the site's *arity*, and a
+//! visitor is given one index for the whole site (a cookie, see
+//! `main.rs`), so they read one storyline from the front door to the
+//! mail they get. A single string is every storyline. What a visitor
+//! did is recorded with the index (`AnswerSubmitted.variant`), so the
+//! storylines can be compared on what people actually did, and nothing
+//! else about them.
+//!
+//! This is done to the YAML before the structs see it: the files are
+//! expanded once per storyline, and everything downstream keeps its one
+//! string per field. Keys that name things to the machine - a field's
+//! `name`, a select's `options`, states, groups, paths - never vary.
+//!
+//! The lint holds every list to the arity, so a storyline cannot have
+//! a hole in it.
+
+use serde_yaml::Value;
+
+/// Keys whose value is one string to a person.
+const TEXT: &[&str] = &["name", "description", "label", "placeholder", "subject", "body", "empty", "title"];
+/// Keys whose value is a list of strings to a person; their storylines
+/// are a list of lists.
+const TEXT_LIST: &[&str] = &["consequence", "encouragements"];
+
+/// Where in a document a key is not text but an identifier: a
+/// requirement's `name` is the stored field, never varied.
+fn is_identifier(path: &[String], key: &str) -> bool {
+    // `path` ends with the key itself, then the item's index, then the
+    // list it is in.
+    key == "name" && path.iter().rev().nth(2).is_some_and(|p| p == "requirements")
+}
+
+fn is_strings(seq: &[Value]) -> bool {
+    !seq.is_empty() && seq.iter().all(|v| v.is_string())
+}
+
+fn is_lists_of_strings(seq: &[Value]) -> bool {
+    !seq.is_empty() && seq.iter().all(|v| v.as_sequence().is_some_and(|s| s.iter().all(|x| x.is_string())))
+}
+
+/// The storyline counts found in a document, with where each was.
+fn counts(value: &Value, path: &mut Vec<String>, out: &mut Vec<(String, usize)>) {
+    match value {
+        Value::Mapping(map) => {
+            for (k, v) in map {
+                let key = k.as_str().unwrap_or_default().to_string();
+                path.push(key.clone());
+                let variants = match v {
+                    Value::Sequence(seq) if TEXT.contains(&key.as_str()) && !is_identifier(path, &key) && is_strings(seq) => Some(seq.len()),
+                    Value::Sequence(seq) if TEXT_LIST.contains(&key.as_str()) && is_lists_of_strings(seq) => Some(seq.len()),
+                    _ => None,
+                };
+                match variants {
+                    Some(n) => out.push((path.join("."), n)),
+                    None => counts(v, path, out),
+                }
+                path.pop();
+            }
+        }
+        Value::Sequence(seq) => {
+            for (i, v) in seq.iter().enumerate() {
+                path.push(i.to_string());
+                counts(v, path, out);
+                path.pop();
+            }
+        }
+        _ => {}
+    }
+}
+
+/// One storyline of a document: every list of texts becomes its `n`th.
+fn pick(value: &mut Value, path: &mut Vec<String>, n: usize) {
+    match value {
+        Value::Mapping(map) => {
+            for (k, v) in map.iter_mut() {
+                let key = k.as_str().unwrap_or_default().to_string();
+                path.push(key.clone());
+                let chosen = match v {
+                    Value::Sequence(seq) if TEXT.contains(&key.as_str()) && !is_identifier(path, &key) && is_strings(seq) => Some(seq[n.min(seq.len() - 1)].clone()),
+                    Value::Sequence(seq) if TEXT_LIST.contains(&key.as_str()) && is_lists_of_strings(seq) => Some(seq[n.min(seq.len() - 1)].clone()),
+                    _ => None,
+                };
+                match chosen {
+                    Some(c) => *v = c,
+                    None => pick(v, path, n),
+                }
+                path.pop();
+            }
+        }
+        Value::Sequence(seq) => {
+            for (i, v) in seq.iter_mut().enumerate() {
+                path.push(i.to_string());
+                pick(v, path, n);
+                path.pop();
+            }
+        }
+        _ => {}
+    }
+}
+
+/// How many storylines these files tell: the length every list of
+/// texts has, or 1 when there is none. A list of another length is an
+/// error that names it.
+pub fn arity(files: &[(String, String)]) -> anyhow::Result<usize> {
+    let mut found: Vec<(String, String, usize)> = Vec::new();
+    for (rel, raw) in files {
+        let Ok(value) = serde_yaml::from_str::<Value>(raw) else { continue };
+        let mut out = Vec::new();
+        counts(&value, &mut Vec::new(), &mut out);
+        found.extend(out.into_iter().map(|(p, n)| (rel.clone(), p, n)));
+    }
+    let arity = found.iter().map(|(_, _, n)| *n).max().unwrap_or(1);
+    if let Some((rel, p, n)) = found.iter().find(|(_, _, n)| *n != arity) {
+        anyhow::bail!("{rel}: {p} tells {n} storyline(s), but the site tells {arity}; every list of texts has the same length");
+    }
+    Ok(arity.max(1))
+}
+
+/// The files as storyline `n` tells them.
+pub fn storyline(files: &[(String, String)], n: usize) -> Vec<(String, String)> {
+    files
+        .iter()
+        .map(|(rel, raw)| {
+            let Ok(mut value) = serde_yaml::from_str::<Value>(raw) else { return (rel.clone(), raw.clone()) };
+            let mut out = Vec::new();
+            counts(&value, &mut Vec::new(), &mut out);
+            if out.is_empty() {
+                return (rel.clone(), raw.clone());
+            }
+            pick(&mut value, &mut Vec::new(), n);
+            (rel.clone(), serde_yaml::to_string(&value).unwrap_or_else(|_| raw.clone()))
+        })
+        .collect()
+}
+
+/// One thing per storyline, and the first as what the machinery reads
+/// when nobody in particular is asking: a seed, a lint, a desk sweep.
+#[derive(Clone, Debug)]
+pub struct Storylines<T> {
+    pub each: Vec<T>,
+}
+
+impl<T> Storylines<T> {
+    pub fn one(item: T) -> Self {
+        Storylines { each: vec![item] }
+    }
+
+    pub fn arity(&self) -> usize {
+        self.each.len().max(1)
+    }
+
+    /// Storyline `n`, or the last for an index from a site that told
+    /// more storylines than this one does.
+    pub fn at(&self, n: usize) -> &T {
+        &self.each[n.min(self.each.len() - 1)]
+    }
+}
+
+impl<T> std::ops::Deref for Storylines<T> {
+    type Target = T;
+    fn deref(&self) -> &T {
+        &self.each[0]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const PAGE: &str = "name: [Vil du hjelpe til?, Trenger naboene deg?]\ndescription: Alle trenger flere hender.\nalternatives:\n  - name: [Bli med, Meld deg]\n    description: [Si hva du kan., Si hva du har lyst på.]\n    consequence: [[Send], [Meld meg]]\n    features:\n      - name: Skjema\n        requirements:\n          - name: [navn]\n            label: [Navn, Hva heter du?]\n          - name: hva\n            type: select\n            options: [Dugnad, Komité]\n";
+
+    #[test]
+    fn every_list_of_texts_is_a_storyline_and_identifiers_are_not() {
+        let files = vec![("index.yaml".to_string(), PAGE.to_string())];
+        assert_eq!(arity(&files).unwrap(), 2);
+        let second = storyline(&files, 1);
+        let v: Value = serde_yaml::from_str(&second[0].1).unwrap();
+        assert_eq!(v["name"], "Trenger naboene deg?");
+        assert_eq!(v["description"], "Alle trenger flere hender.");
+        assert_eq!(v["alternatives"][0]["name"], "Meld deg");
+        assert_eq!(v["alternatives"][0]["consequence"], serde_yaml::to_value(vec!["Meld meg"]).unwrap());
+        // A requirement's name is the stored field: left as written.
+        assert_eq!(v["alternatives"][0]["features"][0]["requirements"][0]["name"], serde_yaml::to_value(vec!["navn"]).unwrap());
+        assert_eq!(v["alternatives"][0]["features"][0]["requirements"][0]["label"], "Hva heter du?");
+        // Options are values people choose from and are stored: untouched.
+        assert_eq!(v["alternatives"][0]["features"][0]["requirements"][1]["options"], serde_yaml::to_value(vec!["Dugnad", "Komité"]).unwrap());
+    }
+
+    #[test]
+    fn a_site_with_no_lists_tells_one_storyline_and_a_short_list_is_refused() {
+        let plain = vec![("index.yaml".to_string(), "name: Hva?\nalternatives:\n  - name: A\n    consequence: [Send]\n".to_string())];
+        assert_eq!(arity(&plain).unwrap(), 1);
+        assert_eq!(storyline(&plain, 0)[0].1, plain[0].1);
+        let uneven = vec![
+            ("index.yaml".to_string(), PAGE.to_string()),
+            ("b.yaml".to_string(), "name: [En, To, Tre]\n".to_string()),
+        ];
+        let err = arity(&uneven).unwrap_err().to_string();
+        assert!(err.contains("b.yaml: name tells 3") && err.contains("tells 2") || err.contains("tells 3"), "{err}");
+    }
+
+    #[test]
+    fn the_first_storyline_is_what_the_machinery_reads() {
+        let s = Storylines { each: vec!["a", "b"] };
+        assert_eq!(*s, "a");
+        assert_eq!(*s.at(1), "b");
+        assert_eq!(*s.at(7), "b");
+        assert_eq!(s.arity(), 2);
+    }
+}