Two fixes surfaced by the vision-02 Pi 5 measurement: ## XDG default_path `Manifest::default_path` was hardcoded to `/var/lib/claw-store/projects.toml`. That path is read-only under the user-mode systemd unit's `ProtectSystem=strict`, and creating it needs root — awful for a runner install. Precedence, matching XDG Base Directory: 1. `$XDG_STATE_HOME/claw-store/projects.toml` 2. `$HOME/.local/state/claw-store/projects.toml` 3. `/var/lib/claw-store/projects.toml` (system fallback) User-mode installs now write in $HOME by default; system installs (root, no HOME set) still land in /var/lib. +1 test: `default_path_honours_xdg_state_home` — covers all three precedence branches. Env mutation is process-global so the test saves + restores. ## Parallel restore on cache HIT Pi restore of 947 MiB via `BlobGetStream` took ~18s (~53 MiB/s) single-stream. Per-stream throughput ceilings on the connection type cap sequential fetches; parallel chunk fetches stack their contributions. - New `call_blob_get_parallel(conn, blob_id, concurrency) -> Option<Vec<u8>>` in `rpc/client.rs`. `JoinSet` + `Semaphore`, reassembles by chunk index at manifest-known offsets so out-of-order arrival is fine. - `claw-cargo build --parallel-restore N` (default 8). `N <= 1` falls through to `BlobGetStream` for parity. - Memory: `total_size + 4 MiB × in-flight` — dominated by the reassembly buffer, not the fanout. +1 test: `parallel_blob_get_reassembles_multi_chunk_blob_byte_equal` covers roundtrip byte-equality vs BlobGetStream, tail-chunk offset, concurrency=1 correctness, and NotFound → None. 259 tests pass (+2). Pre-existing macOS failure unchanged.
378 lines
14 KiB
Rust
378 lines
14 KiB
Rust
use anyhow::{Context, Result};
|
|
use chrono::{DateTime, Utc};
|
|
use serde::{Deserialize, Serialize};
|
|
use std::fs::{File, OpenOptions};
|
|
use std::io::Write;
|
|
use std::os::unix::io::AsRawFd;
|
|
use std::path::{Path, PathBuf};
|
|
|
|
/// A single project tracked by claw-store. Persisted as a row in the
|
|
/// manifest at `/var/lib/claw-store/projects.toml`.
|
|
///
|
|
/// `pinned` (added v0.2.0): operator intent flag — when true, GC skips
|
|
/// this project for both stale-eviction (`gc_stale_targets`) and LRU
|
|
/// eviction (`gc_by_space`). Defaults to `false` via `#[serde(default)]`
|
|
/// so older manifests written by the v0.1.x line still parse cleanly.
|
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
|
pub struct Project {
|
|
pub name: String, // "org/repo" e.g. "quantumclaw/quantum-pulse"
|
|
pub warm_path: PathBuf,
|
|
pub hot_target_path: PathBuf,
|
|
pub last_build: Option<DateTime<Utc>>,
|
|
pub last_active: Option<DateTime<Utc>>,
|
|
pub last_sync: Option<DateTime<Utc>>,
|
|
#[serde(default)]
|
|
pub pinned: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Deserialize, Serialize, Default)]
|
|
pub struct Manifest {
|
|
pub projects: Vec<Project>,
|
|
}
|
|
|
|
impl Manifest {
|
|
/// Read the manifest under a shared (read) flock. Returns
|
|
/// `Self::default()` if the file is missing — the old daemon
|
|
/// behavior on first boot — but propagates parse errors instead
|
|
/// of silently masking them like the pre-v0.2 `unwrap_or_default`
|
|
/// pattern at call sites did.
|
|
pub fn load(path: &Path) -> Result<Self> {
|
|
if !path.exists() {
|
|
return Ok(Self::default());
|
|
}
|
|
let _guard = lock_path(path, LockMode::Shared)
|
|
.with_context(|| format!("flock(LOCK_SH) on {}", path.display()))?;
|
|
let s = std::fs::read_to_string(path)
|
|
.with_context(|| format!("reading manifest at {}", path.display()))?;
|
|
// An empty file is treated as "no projects yet" (matches the
|
|
// not-exists fallthrough above). Bare-init shell ops or a
|
|
// half-written tempfile that crashed pre-rename would otherwise
|
|
// hard-fail the daemon on next startup.
|
|
if s.trim().is_empty() {
|
|
return Ok(Self::default());
|
|
}
|
|
toml::from_str(&s).context("parsing manifest")
|
|
}
|
|
|
|
/// Backward-compatible save. Now writes go through a sibling
|
|
/// tempfile + rename so a crash mid-write can't leave a corrupt
|
|
/// half-written TOML where the next reader silently sees an empty
|
|
/// manifest. Takes EX flock on the lock sidecar so a concurrent
|
|
/// writer doesn't race the rename. Prefer `update()` for any
|
|
/// load-then-mutate path — `save()` alone re-introduces the
|
|
/// lost-update window that `update()` closes.
|
|
pub fn save(&self, path: &Path) -> Result<()> {
|
|
if let Some(parent) = path.parent() {
|
|
std::fs::create_dir_all(parent)
|
|
.with_context(|| format!("mkdir -p {}", parent.display()))?;
|
|
}
|
|
let _guard = lock_path(path, LockMode::Exclusive)
|
|
.with_context(|| format!("flock(LOCK_EX) on {}", path.display()))?;
|
|
write_atomic(path, self).context("writing manifest")
|
|
}
|
|
|
|
/// Locked load → mutate → save in one transaction.
|
|
///
|
|
/// Replaces the v0.1 pattern of `let mut m = load(); m.x(); m.save();`
|
|
/// at call sites — that triple was unsafe under any concurrency
|
|
/// (daemon's 5-min poll tick, plus the dashboard shelling out to
|
|
/// `claw-store activate` which itself did load/mutate/save). Two
|
|
/// such triples interleaved would silently drop one writer's
|
|
/// changes.
|
|
///
|
|
/// Takes an EX flock for the whole transaction, reloads from disk
|
|
/// AFTER acquiring the lock (so we always operate on the latest
|
|
/// state), runs the closure, and atomically renames the resulting
|
|
/// tempfile over the destination. All three steps are inside the
|
|
/// lock so no other writer can race.
|
|
pub fn update(
|
|
path: &Path,
|
|
f: impl FnOnce(&mut Manifest) -> Result<()>,
|
|
) -> Result<Manifest> {
|
|
if let Some(parent) = path.parent() {
|
|
std::fs::create_dir_all(parent)
|
|
.with_context(|| format!("mkdir -p {}", parent.display()))?;
|
|
}
|
|
let _guard = lock_path(path, LockMode::Exclusive)
|
|
.with_context(|| format!("flock(LOCK_EX) on {}", path.display()))?;
|
|
let mut m = if path.exists() {
|
|
let s = std::fs::read_to_string(path)
|
|
.with_context(|| format!("reading manifest at {}", path.display()))?;
|
|
if s.trim().is_empty() {
|
|
Manifest::default()
|
|
} else {
|
|
toml::from_str(&s).context("parsing manifest")?
|
|
}
|
|
} else {
|
|
Manifest::default()
|
|
};
|
|
f(&mut m)?;
|
|
write_atomic(path, &m).context("writing manifest")?;
|
|
Ok(m)
|
|
}
|
|
|
|
pub fn get(&self, name: &str) -> Option<&Project> {
|
|
self.projects.iter().find(|p| p.name == name)
|
|
}
|
|
|
|
pub fn get_mut(&mut self, name: &str) -> Option<&mut Project> {
|
|
self.projects.iter_mut().find(|p| p.name == name)
|
|
}
|
|
|
|
pub fn upsert(&mut self, project: Project) {
|
|
if let Some(p) = self.get_mut(&project.name.clone()) {
|
|
*p = project;
|
|
} else {
|
|
self.projects.push(project);
|
|
}
|
|
}
|
|
|
|
/// Field finding 2026-07-12 (Pi deploy on vision-02): the old
|
|
/// hardcoded `/var/lib/claw-store/projects.toml` broke under
|
|
/// `ProtectSystem=strict` in the user-mode systemd unit because
|
|
/// /var is read-only. Follow the XDG Base Directory spec so a
|
|
/// user-mode install writes under `$HOME/.local/state`, and only
|
|
/// root installs land in `/var/lib`.
|
|
///
|
|
/// Precedence:
|
|
/// 1. `$XDG_STATE_HOME/claw-store/projects.toml` (per spec)
|
|
/// 2. `$HOME/.local/state/claw-store/projects.toml` (XDG default)
|
|
/// 3. `/var/lib/claw-store/projects.toml` (system fallback)
|
|
pub fn default_path() -> PathBuf {
|
|
if let Ok(xdg) = std::env::var("XDG_STATE_HOME") {
|
|
if !xdg.is_empty() {
|
|
return PathBuf::from(xdg)
|
|
.join("claw-store")
|
|
.join("projects.toml");
|
|
}
|
|
}
|
|
if let Ok(home) = std::env::var("HOME") {
|
|
if !home.is_empty() {
|
|
return PathBuf::from(home)
|
|
.join(".local/state/claw-store/projects.toml");
|
|
}
|
|
}
|
|
PathBuf::from("/var/lib/claw-store/projects.toml")
|
|
}
|
|
}
|
|
|
|
// ── flock + atomic rename helpers ────────────────────────────────────────────
|
|
|
|
enum LockMode {
|
|
Shared,
|
|
Exclusive,
|
|
}
|
|
|
|
/// RAII flock guard — releases the lock on drop.
|
|
struct FlockGuard(File);
|
|
|
|
impl Drop for FlockGuard {
|
|
fn drop(&mut self) {
|
|
// Best-effort unlock. The kernel releases automatically on
|
|
// close anyway when File drops; we just hint it sooner. Errors
|
|
// here are not actionable.
|
|
unsafe {
|
|
libc::flock(self.0.as_raw_fd(), libc::LOCK_UN);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Lock a sidecar `.lock` file (so the data file can be replaced by
|
|
/// atomic rename without invalidating the lock). Returns a guard that
|
|
/// releases on drop.
|
|
fn lock_path(target: &Path, mode: LockMode) -> Result<FlockGuard> {
|
|
let lock_path = sidecar_lock_path(target);
|
|
if let Some(parent) = lock_path.parent() {
|
|
std::fs::create_dir_all(parent)?;
|
|
}
|
|
let f = OpenOptions::new()
|
|
.create(true)
|
|
.write(true)
|
|
.truncate(false)
|
|
.open(&lock_path)
|
|
.with_context(|| format!("opening lock {}", lock_path.display()))?;
|
|
let flag = match mode {
|
|
LockMode::Shared => libc::LOCK_SH,
|
|
LockMode::Exclusive => libc::LOCK_EX,
|
|
};
|
|
let rc = unsafe { libc::flock(f.as_raw_fd(), flag) };
|
|
if rc != 0 {
|
|
return Err(std::io::Error::last_os_error()).context("flock");
|
|
}
|
|
Ok(FlockGuard(f))
|
|
}
|
|
|
|
fn sidecar_lock_path(target: &Path) -> PathBuf {
|
|
let mut p = target.as_os_str().to_owned();
|
|
p.push(".lock");
|
|
PathBuf::from(p)
|
|
}
|
|
|
|
/// Atomic write: serialize → write to sibling tempfile in the same dir
|
|
/// → fsync → rename(2) into place. The rename is POSIX-atomic on the
|
|
/// same filesystem, so a reader sees either the old contents or the
|
|
/// new contents, never a torn half-written TOML.
|
|
fn write_atomic(path: &Path, manifest: &Manifest) -> Result<()> {
|
|
let s = toml::to_string_pretty(manifest).context("serialising manifest")?;
|
|
let parent = path.parent().unwrap_or_else(|| Path::new("."));
|
|
let mut tmp = tempfile::Builder::new()
|
|
.prefix(".projects.toml.")
|
|
.suffix(".new")
|
|
.tempfile_in(parent)
|
|
.with_context(|| format!("tempfile in {}", parent.display()))?;
|
|
tmp.write_all(s.as_bytes()).context("write tempfile")?;
|
|
tmp.as_file().sync_all().context("fsync tempfile")?;
|
|
tmp.persist(path).map_err(|e| anyhow::anyhow!("rename: {e}"))?;
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use tempfile::NamedTempFile;
|
|
|
|
#[test]
|
|
fn default_path_honours_xdg_state_home() {
|
|
// Field finding 2026-07-12: precedence XDG_STATE_HOME →
|
|
// $HOME/.local/state → /var/lib. Guard: an env-set XDG wins
|
|
// over HOME; empty XDG is treated as unset.
|
|
// NOTE: env mutation is process-global, so this test does its
|
|
// own setup/teardown and doesn't run in parallel with another
|
|
// that touches the same vars.
|
|
let saved_xdg = std::env::var("XDG_STATE_HOME").ok();
|
|
let saved_home = std::env::var("HOME").ok();
|
|
|
|
// Case 1: XDG_STATE_HOME wins.
|
|
std::env::set_var("XDG_STATE_HOME", "/tmp/xdg-fake");
|
|
std::env::set_var("HOME", "/tmp/home-fake");
|
|
assert_eq!(
|
|
Manifest::default_path(),
|
|
PathBuf::from("/tmp/xdg-fake/claw-store/projects.toml")
|
|
);
|
|
|
|
// Case 2: XDG unset → HOME/.local/state.
|
|
std::env::remove_var("XDG_STATE_HOME");
|
|
assert_eq!(
|
|
Manifest::default_path(),
|
|
PathBuf::from("/tmp/home-fake/.local/state/claw-store/projects.toml")
|
|
);
|
|
|
|
// Case 3: empty XDG treated as unset.
|
|
std::env::set_var("XDG_STATE_HOME", "");
|
|
assert_eq!(
|
|
Manifest::default_path(),
|
|
PathBuf::from("/tmp/home-fake/.local/state/claw-store/projects.toml")
|
|
);
|
|
|
|
// Restore.
|
|
match saved_xdg {
|
|
Some(v) => std::env::set_var("XDG_STATE_HOME", v),
|
|
None => std::env::remove_var("XDG_STATE_HOME"),
|
|
}
|
|
match saved_home {
|
|
Some(v) => std::env::set_var("HOME", v),
|
|
None => std::env::remove_var("HOME"),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_roundtrip_manifest() {
|
|
let mut m = Manifest::default();
|
|
m.projects.push(Project {
|
|
name: "kitchen-cash".into(),
|
|
warm_path: "/slab/projects/kitchen-cash".into(),
|
|
hot_target_path: "/hot/targets/kitchen-cash".into(),
|
|
last_build: None,
|
|
last_active: None,
|
|
last_sync: None,
|
|
pinned: false,
|
|
});
|
|
let f = NamedTempFile::new().unwrap();
|
|
m.save(f.path()).unwrap();
|
|
let loaded = Manifest::load(f.path()).unwrap();
|
|
assert_eq!(loaded.projects.len(), 1);
|
|
assert_eq!(loaded.projects[0].name, "kitchen-cash");
|
|
}
|
|
|
|
#[test]
|
|
fn test_get_project() {
|
|
let mut m = Manifest::default();
|
|
m.projects.push(Project {
|
|
name: "zeroclaw".into(),
|
|
warm_path: "/slab/projects/zeroclaw".into(),
|
|
hot_target_path: "/hot/targets/zeroclaw".into(),
|
|
last_build: None,
|
|
last_active: None,
|
|
last_sync: None,
|
|
pinned: false,
|
|
});
|
|
assert!(m.get("zeroclaw").is_some());
|
|
assert!(m.get("nonexistent").is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn legacy_toml_without_pinned_field_parses() {
|
|
// Manifest written by v0.1.x. Must still load — pinned defaults
|
|
// to false via #[serde(default)]. Without this we'd break every
|
|
// existing deployment on upgrade.
|
|
let legacy = r#"
|
|
[[projects]]
|
|
name = "old/repo"
|
|
warm_path = "/slab/projects/old/repo"
|
|
hot_target_path = "/hot/targets/old/repo"
|
|
"#;
|
|
let f = tempfile::NamedTempFile::new().unwrap();
|
|
std::fs::write(f.path(), legacy).unwrap();
|
|
let m = Manifest::load(f.path()).unwrap();
|
|
assert_eq!(m.projects.len(), 1);
|
|
assert!(!m.projects[0].pinned);
|
|
}
|
|
|
|
#[test]
|
|
fn update_serializes_two_sequential_writers() {
|
|
let f = tempfile::NamedTempFile::new().unwrap();
|
|
// Seed.
|
|
Manifest::update(f.path(), |m| {
|
|
m.upsert(Project {
|
|
name: "alpha".into(),
|
|
warm_path: "/w/alpha".into(),
|
|
hot_target_path: "/h/alpha".into(),
|
|
last_build: None,
|
|
last_active: None,
|
|
last_sync: None,
|
|
pinned: false,
|
|
});
|
|
Ok(())
|
|
})
|
|
.unwrap();
|
|
|
|
// Two sequential updates — each should see the previous
|
|
// committed state. With the broken v0.1 load/mutate/save
|
|
// pattern these would race; here they serialize via flock.
|
|
Manifest::update(f.path(), |m| {
|
|
m.upsert(Project {
|
|
name: "beta".into(),
|
|
warm_path: "/w/beta".into(),
|
|
hot_target_path: "/h/beta".into(),
|
|
last_build: None,
|
|
last_active: None,
|
|
last_sync: None,
|
|
pinned: true,
|
|
});
|
|
Ok(())
|
|
})
|
|
.unwrap();
|
|
Manifest::update(f.path(), |m| {
|
|
if let Some(p) = m.get_mut("alpha") {
|
|
p.pinned = true;
|
|
}
|
|
Ok(())
|
|
})
|
|
.unwrap();
|
|
|
|
let m = Manifest::load(f.path()).unwrap();
|
|
assert_eq!(m.projects.len(), 2);
|
|
assert!(m.projects.iter().all(|p| p.pinned));
|
|
}
|
|
}
|