Merge pull request 'Phase 9 R1: repo-ensure RPC + fleet-health fixes' (#107) from phase-9-r1a-repo-ensure-rpc into main

serve_v2::peer_rpc_addr derives each peer's RPC socket as lan_addr.ip():port+1.
Architect and Tank were bound on their 10G fabric NICs (10.10.0.9/10.10.0.10)
but the peer entries only carry the main-LAN gossip address, causing the
aggregator to time out when probing Tank and present a stale cert to Morpheus.
Changing bind_rpc_lan to the main-LAN addresses (10.0.0.13/10.0.0.14) fixes both.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>

Makefile

@@ -32,7 +32,7 @@ install-systemd:
 install-dashboard:
 	cd dashboard && npm ci && npm run build
 	install -dm755 $(INSTALL_STATIC)
-	cp -r dashboard/dist/. $(INSTALL_STATIC)/
+	cp -r claw-store/static/. $(INSTALL_STATIC)/
 	@echo "Dashboard installed to $(INSTALL_STATIC)"
 
 ## Install node-specific config (NODE=architect|tank)

claw-store/src/cargo_init.rs

@@ -21,8 +21,14 @@ pub fn write_cargo_config(warm_path: &Path, hot_target_path: &Path) -> Result<()
         String::new()
     };
 
+    // Drop any leading copies of our own marker comment before stripping the
+    // [build] section — it sits above the [build] header, outside the range
+    // strip_section tracks, so without this it would survive every
+    // regenerate cycle and duplicate one more time.
+    let existing = strip_leading_marker(&existing);
+
     // Remove old [build] block (from its header to the next section or EOF).
-    let stripped = strip_section(&existing, "build");
+    let stripped = strip_section(existing, "build");
 
     let new_content = format!(
         "# claw-store managed — do not edit manually\n\
@@ -37,6 +43,21 @@ pub fn write_cargo_config(warm_path: &Path, hot_target_path: &Path) -> Result<()
         .with_context(|| format!("writing {}", config_path.display()))
 }
 
+const MANAGED_MARKER: &str = "# claw-store managed — do not edit manually";
+
+/// Strip leading copies of `MANAGED_MARKER`, one per line, from the start of `src`.
+fn strip_leading_marker(src: &str) -> &str {
+    let mut rest = src;
+    while let Some(line_end) = rest.find('\n') {
+        if rest[..line_end].trim() == MANAGED_MARKER {
+            rest = &rest[line_end + 1..];
+        } else {
+            break;
+        }
+    }
+    rest
+}
+
 /// Remove a TOML section `[name]` and all its key=value lines from `src`,
 /// stopping at the next `[section]` header or EOF.
 fn strip_section(src: &str, name: &str) -> String {
@@ -102,6 +123,46 @@ mod tests {
         assert!(verify_cargo_config(&warm, &hot).unwrap());
     }
 
+    #[test]
+    fn test_write_cargo_config_idempotent_no_duplicate_marker() {
+        let dir = TempDir::new().unwrap();
+        let warm = dir.path().join("proj");
+        std::fs::create_dir_all(&warm).unwrap();
+        let hot: std::path::PathBuf = "/hot/targets/proj".into();
+
+        write_cargo_config(&warm, &hot).unwrap();
+        write_cargo_config(&warm, &hot).unwrap();
+        write_cargo_config(&warm, &hot).unwrap();
+
+        let content = std::fs::read_to_string(warm.join(".cargo/config.toml")).unwrap();
+        assert_eq!(content.matches("claw-store managed").count(), 1);
+    }
+
+    #[test]
+    fn test_write_cargo_config_heals_existing_duplicate_marker() {
+        let dir = TempDir::new().unwrap();
+        let warm = dir.path().join("proj");
+        let cargo_dir = warm.join(".cargo");
+        std::fs::create_dir_all(&cargo_dir).unwrap();
+        std::fs::write(
+            cargo_dir.join("config.toml"),
+            "# claw-store managed — do not edit manually\n\
+             [build]\n\
+             target-dir = \"/hot/targets/proj\"\n\
+             # claw-store managed — do not edit manually\n\
+             [env]\n\
+             FOO = \"bar\"\n",
+        )
+        .unwrap();
+
+        let hot: std::path::PathBuf = "/hot/targets/proj".into();
+        write_cargo_config(&warm, &hot).unwrap();
+
+        let content = std::fs::read_to_string(cargo_dir.join("config.toml")).unwrap();
+        assert_eq!(content.matches("claw-store managed").count(), 1);
+        assert!(content.contains("FOO = \"bar\""));
+    }
+
     #[test]
     fn test_verify_cargo_config_detects_missing() {
         let dir = TempDir::new().unwrap();

config/architect.toml

@@ -19,6 +19,33 @@ archive_path = "/data/archive"
 zfs_dataset = "data/archive"
 retain_weeks = 12
 
+[cluster]
+zone = "fabric-10g"
+bind_lan = "10.0.0.13:7701"
+prom_bind = "0.0.0.0:7703"
+bind_rpc_lan = "10.0.0.13:7702"
+bind_rpc_tailscale = "100.104.171.32:7702"
+blob_store_root = "/home/osobh/clawstor-deploy/data"
+
+[[cluster.peers]]
+name = "tank"
+zone = "fabric-10g"
+lan_addr = "10.0.0.14:7701"
+rpc_lan_addr = "10.10.0.10:7702"
+tailscale_addr = "100.108.129.81:7702"
+
+[[cluster.peers]]
+name = "morpheus"
+zone = "lan-1g"
+lan_addr = "10.0.0.5:7701"
+rpc_lan_addr = "10.0.0.5:7702"
+tailscale_addr = "100.123.224.84:7702"
+
+[cluster.tls]
+ca_cert = "/home/osobh/clawstor-deploy/tls/ca.crt"
+node_cert = "/home/osobh/clawstor-deploy/tls/node.crt"
+node_key = "/home/osobh/clawstor-deploy/tls/node.key"
+
 [replication]
 receive_from_peer = true
 peer_user = "osobh"

config/morpheus.toml

@@ -0,0 +1,43 @@
+[node]
+name = "morpheus"
+role = "secondary"
+
+[hot]
+path = "/hot/targets"
+max_gb = 80
+stale_hours = 48
+
+[warm]
+projects_path = "/slab/projects"
+zfs_dataset = "none"
+snapshot_retain_hours = 24
+snapshot_retain_days = 7
+snapshot_retain_weeks = 4
+
+[cluster]
+zone = "lan-1g"
+# Morpheus has no direct 10G to Architect/Tank — use main LAN for all traffic
+bind_lan = "10.0.0.5:7701"
+prom_bind = "0.0.0.0:7703"
+bind_rpc_lan = "10.0.0.5:7702"
+bind_rpc_tailscale = "100.123.224.84:7702"
+blob_store_root = "/home/osobh/clawstor-deploy/data"
+
+[[cluster.peers]]
+name = "architect"
+zone = "fabric-10g"
+lan_addr = "10.0.0.13:7701"
+rpc_lan_addr = "10.0.0.13:7702"
+tailscale_addr = "100.104.171.32:7702"
+
+[[cluster.peers]]
+name = "tank"
+zone = "fabric-10g"
+lan_addr = "10.0.0.14:7701"
+rpc_lan_addr = "10.0.0.14:7702"
+tailscale_addr = "100.108.129.81:7702"
+
+[cluster.tls]
+ca_cert = "/home/osobh/clawstor-deploy/tls/ca.crt"
+node_cert = "/home/osobh/clawstor-deploy/tls/node.crt"
+node_key = "/home/osobh/clawstor-deploy/tls/node.key"

config/tank.toml

@@ -14,13 +14,35 @@ snapshot_retain_hours = 24
 snapshot_retain_days = 7
 snapshot_retain_weeks = 4
 
+[cluster]
+zone = "fabric-10g"
+bind_lan = "10.0.0.14:7701"
+prom_bind = "0.0.0.0:7703"
+bind_rpc_lan = "10.0.0.14:7702"
+bind_rpc_tailscale = "100.108.129.81:7702"
+blob_store_root = "/home/osobh/clawstor-deploy/data"
+
+[[cluster.peers]]
+name = "architect"
+zone = "fabric-10g"
+lan_addr = "10.0.0.13:7701"
+rpc_lan_addr = "10.10.0.9:7702"
+tailscale_addr = "100.104.171.32:7702"
+
+[[cluster.peers]]
+name = "morpheus"
+zone = "lan-1g"
+lan_addr = "10.0.0.5:7701"
+rpc_lan_addr = "10.0.0.5:7702"
+tailscale_addr = "100.123.224.84:7702"
+
+[cluster.tls]
+ca_cert = "/home/osobh/clawstor-deploy/tls/ca.crt"
+node_cert = "/home/osobh/clawstor-deploy/tls/node.crt"
+node_key = "/home/osobh/clawstor-deploy/tls/node.key"
+
 [replication]
-# 10.10.0.9 is architect-fab-tank — the dedicated 10G fabric link between
-# the two nodes. Intentionally used for replication to maximise bandwidth;
-# architect's primary LAN address is 10.0.0.13.
 send_to_host = "10.10.0.9"
 send_to_user = "osobh"
 cold_dataset_on_peer = "data/archive/tank-projects"
-# nightly_at is reserved for future use; replication schedule is currently
-# controlled by the claw-store-replicate.timer systemd unit.
 nightly_at = "03:30"

systemd/claw-fuse.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=clawstor FUSE mount
+After=claw-store.service
+Requires=claw-store.service
+
+[Service]
+Type=simple
+User=osobh
+Environment=PATH=/home/osobh/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ExecStartPre=/bin/mkdir -p /home/osobh/clawstor-mount
+ExecStart=/usr/local/bin/claw-fuse --data-dir /home/osobh/clawstor-deploy/data --mount /home/osobh/clawstor-mount
+ExecStop=/bin/fusermount -u /home/osobh/clawstor-mount
+Restart=on-failure
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

systemd/claw-store-serve.service

@@ -11,7 +11,7 @@ Wants=network-online.target
 [Service]
 Type=simple
 User=osobh
-ExecStart=/usr/local/bin/claw-store serve --port 7700 --static-dir /usr/share/claw-store/static
+ExecStart=/usr/local/bin/claw-store serve --port 7700 --static-dir /usr/share/claw-store/static --v2-static-dir /usr/share/claw-store/v2
 Restart=on-failure
 RestartSec=15
 Environment=RUST_LOG=info

systemd/claw-store.service

@@ -1,15 +1,19 @@
 [Unit]
-Description=claw-store fleet storage daemon
+Description=clawstor cluster daemon (gossip + QUIC blob store + ZFS snapshots)
-After=zfs-mount.service network.target
+After=network-online.target
-Wants=zfs-mount.service
+Wants=network-online.target
 
 [Service]
 Type=simple
 User=osobh
+Environment=PATH=/home/osobh/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+Environment=RUST_LOG=info
 ExecStart=/usr/local/bin/claw-store daemon
 Restart=on-failure
-RestartSec=30
+RestartSec=15
-Environment=RUST_LOG=info
+TimeoutStopSec=60
+ProtectSystem=strict
+ReadWritePaths=/var/lib/claw-store /hot/targets /slab/projects /home/osobh/clawstor-deploy
 
 [Install]
 WantedBy=multi-user.target