Milestone 2: persistent blob store, add/materialize/gc
iroh-blobs 0.35 fs store under <store_dir>/blobs. Add imports files or whole directory trees (as Collections, deterministic order), Materialize exports them with a real FICLONE reflink attempt and streaming-copy fallback, Gc is an explicit mark-and-sweep rooted at the pins. Pins, trusted peers and hash formats persist in meta.json (atomic writes). Store reads run on a LocalPool because iroh-blobs entry readers are not Send. Integration tests drive the real daemon binary: directory round trip byte-identical, gc keeps pinned/drops unpinned, reflink verified on the repo's own filesystem (XFS), plus a 32-case proptest round trip. Dependencies: iroh-blobs =0.35.0 (the store itself; pinned per spec), iroh-io (AsyncSliceReader traits to read store entries), reflink-copy (FICLONE with copy fallback, per spec), proptest (dev-only, round-trip property test). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Bendik Lynghaug
co-authored by
Claude Fable 5
Claude Fable 5
parent
4e1a95613b
commit
e256f73439
@@ -0,0 +1,183 @@
|
||||
//! Persistent daemon metadata: pins, known formats, trusted peers.
|
||||
//!
|
||||
//! Stored as a single JSON file (`meta.json`) in the store directory —
|
||||
//! the spec allows JSON for the MVP and the data is tiny and human
|
||||
//! auditable. Writes go through a temp file + rename so a crash never
|
||||
//! leaves a torn file.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use varde_proto::PinPolicy;
|
||||
|
||||
/// Whether a hash names a single blob or a HashSeq (directory root).
|
||||
///
|
||||
/// Recorded at import/pin time because the bytes alone don't say; a
|
||||
/// materialize of an unknown hash defaults to `Raw`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum StoredFormat {
|
||||
Raw,
|
||||
HashSeq,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PinRecord {
|
||||
pub policy: PinPolicy,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
struct MetaState {
|
||||
/// Pinned root hashes (hex) and their policies.
|
||||
#[serde(default)]
|
||||
pins: BTreeMap<String, PinRecord>,
|
||||
/// Known formats of hashes we imported or fetched (hex).
|
||||
#[serde(default)]
|
||||
formats: BTreeMap<String, StoredFormat>,
|
||||
/// Trusted peer NodeIds (z-base-32).
|
||||
#[serde(default)]
|
||||
trusted_peers: BTreeSet<String>,
|
||||
}
|
||||
|
||||
/// Handle to the metadata file. Cheap to share behind an `Arc`.
|
||||
#[derive(Debug)]
|
||||
pub struct Meta {
|
||||
path: PathBuf,
|
||||
state: Mutex<MetaState>,
|
||||
}
|
||||
|
||||
impl Meta {
|
||||
/// Load metadata from `store_dir/meta.json`, starting empty if absent.
|
||||
pub fn load(store_dir: &Path) -> Result<Meta> {
|
||||
let path = store_dir.join("meta.json");
|
||||
let state = match std::fs::read_to_string(&path) {
|
||||
Ok(text) => serde_json::from_str(&text)
|
||||
.with_context(|| format!("parsing {}", path.display()))?,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => MetaState::default(),
|
||||
Err(e) => return Err(e).with_context(|| format!("reading {}", path.display())),
|
||||
};
|
||||
Ok(Meta {
|
||||
path,
|
||||
state: Mutex::new(state),
|
||||
})
|
||||
}
|
||||
|
||||
fn mutate<R>(&self, f: impl FnOnce(&mut MetaState) -> R) -> Result<R> {
|
||||
let mut state = self.state.lock().expect("meta lock poisoned");
|
||||
let result = f(&mut state);
|
||||
// Persist inside the lock so concurrent mutations can't write
|
||||
// stale snapshots over each other. The file is tiny.
|
||||
let json = serde_json::to_string_pretty(&*state)?;
|
||||
let tmp = self.path.with_extension("json.tmp");
|
||||
std::fs::write(&tmp, json).with_context(|| format!("writing {}", tmp.display()))?;
|
||||
std::fs::rename(&tmp, &self.path)
|
||||
.with_context(|| format!("renaming into {}", self.path.display()))?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Create or update a pin.
|
||||
pub fn pin(&self, hash: &str, policy: PinPolicy) -> Result<()> {
|
||||
self.mutate(|s| {
|
||||
s.pins.insert(hash.to_string(), PinRecord { policy });
|
||||
})
|
||||
}
|
||||
|
||||
/// Remove a pin. Returns false if there was none.
|
||||
pub fn unpin(&self, hash: &str) -> Result<bool> {
|
||||
self.mutate(|s| s.pins.remove(hash).is_some())
|
||||
}
|
||||
|
||||
pub fn is_pinned(&self, hash: &str) -> bool {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("meta lock poisoned")
|
||||
.pins
|
||||
.contains_key(hash)
|
||||
}
|
||||
|
||||
/// Snapshot of all pins as (hash, record).
|
||||
pub fn pins(&self) -> Vec<(String, PinRecord)> {
|
||||
let state = self.state.lock().expect("meta lock poisoned");
|
||||
state
|
||||
.pins
|
||||
.iter()
|
||||
.map(|(h, r)| (h.clone(), r.clone()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Record the format of a hash we learned about.
|
||||
pub fn record_format(&self, hash: &str, format: StoredFormat) -> Result<()> {
|
||||
self.mutate(|s| {
|
||||
s.formats.insert(hash.to_string(), format);
|
||||
})
|
||||
}
|
||||
|
||||
/// Best knowledge of the format of `hash`; defaults to `Raw`.
|
||||
pub fn format_of(&self, hash: &str) -> StoredFormat {
|
||||
let state = self.state.lock().expect("meta lock poisoned");
|
||||
state
|
||||
.formats
|
||||
.get(hash)
|
||||
.copied()
|
||||
.unwrap_or(StoredFormat::Raw)
|
||||
}
|
||||
|
||||
/// Add a trusted peer. Returns false if it was already trusted.
|
||||
pub fn trust_peer(&self, node_id: &str) -> Result<bool> {
|
||||
self.mutate(|s| s.trusted_peers.insert(node_id.to_string()))
|
||||
}
|
||||
|
||||
/// Remove a trusted peer. Returns false if it was not trusted.
|
||||
pub fn untrust_peer(&self, node_id: &str) -> Result<bool> {
|
||||
self.mutate(|s| s.trusted_peers.remove(node_id))
|
||||
}
|
||||
|
||||
pub fn trusted_peers(&self) -> BTreeSet<String> {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("meta lock poisoned")
|
||||
.trusted_peers
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub fn is_trusted(&self, node_id: &str) -> bool {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("meta lock poisoned")
|
||||
.trusted_peers
|
||||
.contains(node_id)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn survives_reload() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let meta = Meta::load(dir.path()).unwrap();
|
||||
meta.pin("abc", PinPolicy { open_lan: true }).unwrap();
|
||||
meta.record_format("abc", StoredFormat::HashSeq).unwrap();
|
||||
meta.trust_peer("node1").unwrap();
|
||||
|
||||
let meta = Meta::load(dir.path()).unwrap();
|
||||
assert!(meta.is_pinned("abc"));
|
||||
assert_eq!(meta.format_of("abc"), StoredFormat::HashSeq);
|
||||
assert!(meta.is_trusted("node1"));
|
||||
assert_eq!(meta.format_of("unknown"), StoredFormat::Raw);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unpin_reports_absence() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let meta = Meta::load(dir.path()).unwrap();
|
||||
assert!(!meta.unpin("nope").unwrap());
|
||||
meta.pin("h", PinPolicy::default()).unwrap();
|
||||
assert!(meta.unpin("h").unwrap());
|
||||
assert!(!meta.is_pinned("h"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user