Event-sourced applicant/subscriber/project aggregates, generalized resources
Deploy / deploy (push) Failing after 3s
Deploy / deploy (push) Failing after 3s
Replaces the free-string, direct-KV-mutate state model in answers.rs
with a proper event log (events/store.rs, JetStream-backed, CAS via
expected_last_subject_sequence) and three pure state machines
(aggregates/{applicant,subscriber,project}.rs). Closes a real
lost-update race in the old transition_answer (concurrent decisions on
the same item could both win, publishing contradictory events). KV
buckets become best-effort read-model projections, not the source of
truth. Content-declared transition targets are now validated at
load/reload time against the real compiled transition tables, not
accepted as arbitrary strings.
Buckets renamed to describe their content, not their relation to the
app (portal_applicants -> applicants, etc); "inquiry" folded into a
richer "project" concept.
ResourceSpec generalized beyond a single KV bucket: Kv | GiteaStarred |
GiteaOrgRepos | Url sources, with an optional jq filter (via the jaq
crate) to shape live data for the frontend. Url source is SSRF-guarded
(https-only, rejects loopback/private/link-local, real DNS resolve).
New headless question_lint binary (validates content against compiled
transition tables with no NATS/OIDC/server involved) and a one-time
backfill_events binary (dry-run by default) for migrating existing KV
data onto the new event log.
Questions get an optional `responsible` contact plus a lightweight
"report this question" action.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
213b1130bb
commit
ebf4bf91b3
+273
-36
@@ -1,24 +1,30 @@
|
||||
//! Generic, authorized reads of NATS KV data declared in content
|
||||
//! Generic, authorized reads of live data declared in content
|
||||
//! (`content::ResourceSpec`, on a `Feature`). The one property that
|
||||
//! matters: the bucket/key a resource reads is never a client-supplied
|
||||
//! parameter, only something the server resolves from its own
|
||||
//! trusted, YAML-loaded content - the client only ever names a
|
||||
//! question id + feature name, so it can't probe an arbitrary bucket
|
||||
//! by just naming it.
|
||||
//! matters: what a resource reads is never a client-supplied parameter,
|
||||
//! only something the server resolves from its own trusted, YAML-loaded
|
||||
//! content - the client only ever names a question id + feature name
|
||||
//! (plus, now, whatever form-field values it already has, forwarded as
|
||||
//! query params), never a bucket/URL directly.
|
||||
|
||||
use leptos::prelude::*;
|
||||
|
||||
/// Fetches the live data for `question_id`'s `alternative`'s
|
||||
/// `feature_name` feature. Fails closed: a resource with neither
|
||||
/// `public: true` nor `requires_group` set is unreachable, not "open"
|
||||
/// by omission.
|
||||
/// by omission. `params` are the requesting form's current field
|
||||
/// values (if any) - forwarded as query parameters to a live external
|
||||
/// fetch (`GiteaStarred`/`GiteaOrgRepos`/`Url`), the mechanism behind
|
||||
/// "a resource parameterized by other form fields"; a `Kv` resource
|
||||
/// ignores them entirely, same as today.
|
||||
#[server]
|
||||
pub async fn get_resource(
|
||||
question_id: String,
|
||||
alternative: String,
|
||||
feature_name: String,
|
||||
params: std::collections::HashMap<String, String>,
|
||||
) -> Result<serde_json::Value, ServerFnError> {
|
||||
use crate::auth::{User, SESSION_USER_KEY};
|
||||
use crate::content::ResourceSource;
|
||||
use crate::server::AppState;
|
||||
|
||||
let state = expect_context::<AppState>();
|
||||
@@ -66,39 +72,270 @@ pub async fn get_resource(
|
||||
}
|
||||
}
|
||||
|
||||
let store = state
|
||||
.jetstream
|
||||
.get_key_value(&resource.bucket)
|
||||
.await
|
||||
.map_err(|e| ServerFnError::new(format!("resource bucket unavailable: {e}")))?;
|
||||
|
||||
match &resource.key {
|
||||
Some(key) => {
|
||||
let bytes = store
|
||||
.get(key)
|
||||
let value = match &resource.source {
|
||||
ResourceSource::Kv { bucket } => {
|
||||
let store = state
|
||||
.jetstream
|
||||
.get_key_value(bucket)
|
||||
.await
|
||||
.map_err(|e| ServerFnError::new(e.to_string()))?
|
||||
.ok_or_else(|| ServerFnError::new("resource key not found"))?;
|
||||
serde_json::from_slice(&bytes).map_err(|e| ServerFnError::new(e.to_string()))
|
||||
}
|
||||
None => {
|
||||
use futures::TryStreamExt;
|
||||
let keys: Vec<String> = store
|
||||
.keys()
|
||||
.await
|
||||
.map_err(|e| ServerFnError::new(e.to_string()))?
|
||||
.try_collect()
|
||||
.await
|
||||
.map_err(|e| ServerFnError::new(e.to_string()))?;
|
||||
let mut items = Vec::new();
|
||||
for key in keys {
|
||||
if let Ok(Some(bytes)) = store.get(&key).await {
|
||||
if let Ok(value) = serde_json::from_slice::<serde_json::Value>(&bytes) {
|
||||
items.push(value);
|
||||
.map_err(|e| ServerFnError::new(format!("resource bucket unavailable: {e}")))?;
|
||||
match &resource.key {
|
||||
Some(key) => {
|
||||
let bytes = store
|
||||
.get(key)
|
||||
.await
|
||||
.map_err(|e| ServerFnError::new(e.to_string()))?
|
||||
.ok_or_else(|| ServerFnError::new("resource key not found"))?;
|
||||
serde_json::from_slice(&bytes).map_err(|e| ServerFnError::new(e.to_string()))?
|
||||
}
|
||||
None => {
|
||||
use futures::TryStreamExt;
|
||||
let keys: Vec<String> = store
|
||||
.keys()
|
||||
.await
|
||||
.map_err(|e| ServerFnError::new(e.to_string()))?
|
||||
.try_collect()
|
||||
.await
|
||||
.map_err(|e| ServerFnError::new(e.to_string()))?;
|
||||
let mut items = Vec::new();
|
||||
for key in keys {
|
||||
if let Ok(Some(bytes)) = store.get(&key).await {
|
||||
if let Ok(value) = serde_json::from_slice::<serde_json::Value>(&bytes) {
|
||||
items.push(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
serde_json::Value::Array(items)
|
||||
}
|
||||
}
|
||||
Ok(serde_json::Value::Array(items))
|
||||
}
|
||||
ResourceSource::GiteaStarred { username } => {
|
||||
fetch_gitea_json(&state, &format!("/api/v1/users/{username}/starred"), ¶ms).await?
|
||||
}
|
||||
ResourceSource::GiteaOrgRepos { org } => {
|
||||
fetch_gitea_json(&state, &format!("/api/v1/orgs/{org}/repos"), ¶ms).await?
|
||||
}
|
||||
ResourceSource::Url { url } => fetch_url_json(url, ¶ms).await?,
|
||||
};
|
||||
|
||||
match &resource.jq {
|
||||
Some(filter) => apply_jq(filter, &value),
|
||||
None => Ok(value),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches JSON from a path on the same Gitea instance content is
|
||||
/// loaded from (`AppState.gitea_base`), attaching `GITEA_API_TOKEN` if
|
||||
/// set - confirmed live against `project.uhhm.no` that at least the
|
||||
/// starred-repos endpoint 401s unauthenticated, unlike the repo
|
||||
/// -contents/repo-info endpoints `content.rs` already calls anonymously.
|
||||
#[cfg(feature = "ssr")]
|
||||
async fn fetch_gitea_json(
|
||||
state: &crate::server::AppState,
|
||||
path: &str,
|
||||
params: &std::collections::HashMap<String, String>,
|
||||
) -> Result<serde_json::Value, ServerFnError> {
|
||||
let mut url = url::Url::parse(&format!("{}{}", state.gitea_base, path))
|
||||
.map_err(|e| ServerFnError::new(e.to_string()))?;
|
||||
for (k, v) in params {
|
||||
url.query_pairs_mut().append_pair(k, v);
|
||||
}
|
||||
let client = openidconnect::reqwest::Client::new();
|
||||
let mut req = client.get(url.clone());
|
||||
if let Ok(token) = std::env::var("GITEA_API_TOKEN") {
|
||||
if !token.is_empty() {
|
||||
req = req.header("Authorization", format!("token {token}"));
|
||||
}
|
||||
}
|
||||
let text = req
|
||||
.send()
|
||||
.await
|
||||
.and_then(|r| r.error_for_status())
|
||||
.map_err(|e| ServerFnError::new(format!("fetching {url}: {e}")))?
|
||||
.text()
|
||||
.await
|
||||
.map_err(|e| ServerFnError::new(e.to_string()))?;
|
||||
serde_json::from_str(&text).map_err(|e| ServerFnError::new(e.to_string()))
|
||||
}
|
||||
|
||||
/// Fetches JSON from an arbitrary content-declared URL
|
||||
/// (`content::ResourceSource::Url`). Content is trusted today (same
|
||||
/// org owns both this app and its content repo), but a server-side
|
||||
/// fetch of a content-supplied URL is real SSRF surface the moment
|
||||
/// that stops being true - `assert_public_https_url` fails closed on
|
||||
/// anything that isn't `https://` and actually resolves to a public
|
||||
/// address, not just a syntactically-plausible one.
|
||||
#[cfg(feature = "ssr")]
|
||||
async fn fetch_url_json(
|
||||
url: &str,
|
||||
params: &std::collections::HashMap<String, String>,
|
||||
) -> Result<serde_json::Value, ServerFnError> {
|
||||
let mut parsed = assert_public_https_url(url)
|
||||
.await
|
||||
.map_err(|e| ServerFnError::new(e.to_string()))?;
|
||||
for (k, v) in params {
|
||||
parsed.query_pairs_mut().append_pair(k, v);
|
||||
}
|
||||
let client = openidconnect::reqwest::Client::new();
|
||||
let text = client
|
||||
.get(parsed.clone())
|
||||
.send()
|
||||
.await
|
||||
.and_then(|r| r.error_for_status())
|
||||
.map_err(|e| ServerFnError::new(format!("fetching {parsed}: {e}")))?
|
||||
.text()
|
||||
.await
|
||||
.map_err(|e| ServerFnError::new(e.to_string()))?;
|
||||
serde_json::from_str(&text).map_err(|e| ServerFnError::new(e.to_string()))
|
||||
}
|
||||
|
||||
/// `https://` only, and only if every address the host resolves to is
|
||||
/// public - rejects loopback (127.0.0.1, ::1), RFC1918/link-local
|
||||
/// private ranges, and unspecified/multicast addresses. Resolves via
|
||||
/// real DNS (not just a syntax check) so a hostname that *points at* an
|
||||
/// internal address is caught too, not just a literal internal IP
|
||||
/// spelled out directly.
|
||||
#[cfg(feature = "ssr")]
|
||||
async fn assert_public_https_url(url: &str) -> anyhow::Result<url::Url> {
|
||||
let parsed = url::Url::parse(url)?;
|
||||
if parsed.scheme() != "https" {
|
||||
anyhow::bail!("only https:// resource urls are allowed");
|
||||
}
|
||||
let host = parsed
|
||||
.host_str()
|
||||
.ok_or_else(|| anyhow::anyhow!("resource url has no host"))?;
|
||||
let port = parsed.port_or_known_default().unwrap_or(443);
|
||||
let mut resolved = false;
|
||||
for addr in tokio::net::lookup_host((host, port))
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("resolving resource host {host}: {e}"))?
|
||||
{
|
||||
resolved = true;
|
||||
if !is_public_ip(&addr.ip()) {
|
||||
anyhow::bail!("resource host {host} resolves to a non-public address");
|
||||
}
|
||||
}
|
||||
if !resolved {
|
||||
anyhow::bail!("resource host {host} did not resolve to any address");
|
||||
}
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
#[cfg(feature = "ssr")]
|
||||
fn is_public_ip(ip: &std::net::IpAddr) -> bool {
|
||||
match ip {
|
||||
std::net::IpAddr::V4(v4) => {
|
||||
!(v4.is_private()
|
||||
|| v4.is_loopback()
|
||||
|| v4.is_link_local()
|
||||
|| v4.is_multicast()
|
||||
|| v4.is_broadcast()
|
||||
|| v4.is_unspecified()
|
||||
|| v4.is_documentation())
|
||||
}
|
||||
std::net::IpAddr::V6(v6) => {
|
||||
let is_unique_local = (v6.segments()[0] & 0xfe00) == 0xfc00;
|
||||
let is_link_local = (v6.segments()[0] & 0xffc0) == 0xfe80;
|
||||
!(v6.is_loopback()
|
||||
|| v6.is_unspecified()
|
||||
|| v6.is_multicast()
|
||||
|| is_unique_local
|
||||
|| is_link_local)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs `filter_src` (a jq filter) over `input`, collecting every value
|
||||
/// it yields into a JSON array - e.g. `.[] | {name, url: .html_url}`
|
||||
/// over a Gitea repo list yields one shaped object per repo. Pure Rust
|
||||
/// via the `jaq` crate family, no shell-out to a `jq` binary.
|
||||
#[cfg(feature = "ssr")]
|
||||
pub fn apply_jq(
|
||||
filter_src: &str,
|
||||
input: &serde_json::Value,
|
||||
) -> Result<serde_json::Value, ServerFnError> {
|
||||
use jaq_core::load::{Arena, File, Loader};
|
||||
use jaq_core::{data, unwrap_valr, Compiler, Ctx, Vars};
|
||||
use jaq_json::{read, Val};
|
||||
|
||||
let input_text = input.to_string();
|
||||
let input_val = read::parse_single(input_text.as_bytes())
|
||||
.map_err(|e| ServerFnError::new(format!("internal: re-parsing json for jq: {e:?}")))?;
|
||||
|
||||
let program = File { code: filter_src, path: () };
|
||||
let defs = jaq_core::defs().chain(jaq_std::defs()).chain(jaq_json::defs());
|
||||
let funs = jaq_core::funs().chain(jaq_std::funs()).chain(jaq_json::funs());
|
||||
let loader = Loader::new(defs);
|
||||
let arena = Arena::default();
|
||||
let modules = loader
|
||||
.load(&arena, program)
|
||||
.map_err(|e| ServerFnError::new(format!("jq filter parse error: {e:?}")))?;
|
||||
let filter = Compiler::default()
|
||||
.with_funs(funs)
|
||||
.compile(modules)
|
||||
.map_err(|e| ServerFnError::new(format!("jq filter compile error: {e:?}")))?;
|
||||
let ctx = Ctx::<data::JustLut<Val>>::new(&filter.lut, Vars::new([]));
|
||||
|
||||
let mut outputs = Vec::new();
|
||||
for result in filter.id.run((ctx, input_val)).map(unwrap_valr) {
|
||||
let val = result.map_err(|e| ServerFnError::new(format!("jq filter runtime error: {e:?}")))?;
|
||||
let text = val.to_string();
|
||||
let json: serde_json::Value = serde_json::from_str(&text)
|
||||
.map_err(|e| ServerFnError::new(format!("internal: parsing jq output: {e}")))?;
|
||||
outputs.push(json);
|
||||
}
|
||||
Ok(serde_json::Value::Array(outputs))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A canned, Gitea-API-shaped fixture - the same fields the real
|
||||
/// `/users/{username}/starred` endpoint returns - run through the
|
||||
/// filter `index.yaml`'s "What we've built" resource actually
|
||||
/// declares, confirming the `jaq` integration produces the shape
|
||||
/// the frontend showcase card expects.
|
||||
#[test]
|
||||
fn jq_shapes_gitea_repo_list_for_the_showcase() {
|
||||
let input = serde_json::json!([
|
||||
{
|
||||
"name": "cnats",
|
||||
"description": "A NATS-backed chat client",
|
||||
"html_url": "https://project.uhhm.no/bl/cnats",
|
||||
"stargazers_count": 3,
|
||||
"private": false
|
||||
},
|
||||
{
|
||||
"name": "portal",
|
||||
"description": "This app",
|
||||
"html_url": "https://project.uhhm.no/uhhm/portal",
|
||||
"stargazers_count": 1,
|
||||
"private": false
|
||||
}
|
||||
]);
|
||||
let filter = ".[] | {name: .name, description: .description, url: .html_url, stars: .stargazers_count}";
|
||||
|
||||
let shaped = apply_jq(filter, &input).expect("filter runs");
|
||||
let items = shaped.as_array().expect("array output");
|
||||
assert_eq!(items.len(), 2);
|
||||
assert_eq!(items[0]["name"], "cnats");
|
||||
assert_eq!(items[0]["url"], "https://project.uhhm.no/bl/cnats");
|
||||
assert_eq!(items[0]["stars"], 3);
|
||||
// The filter never mentions `private` - confirms shaping
|
||||
// actually drops fields, not just passes the object through.
|
||||
assert!(items[0].get("private").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_public_ip_rejects_loopback_and_private() {
|
||||
let loopback: std::net::IpAddr = "127.0.0.1".parse().unwrap();
|
||||
let private: std::net::IpAddr = "10.0.0.5".parse().unwrap();
|
||||
let link_local: std::net::IpAddr = "169.254.1.1".parse().unwrap();
|
||||
let public: std::net::IpAddr = "1.1.1.1".parse().unwrap();
|
||||
assert!(!is_public_ip(&loopback));
|
||||
assert!(!is_public_ip(&private));
|
||||
assert!(!is_public_ip(&link_local));
|
||||
assert!(is_public_ip(&public));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user