Pilot findings: 5 real-world fixes from the 2026-07-12 deploy

Bundles the profile→dir bug (PR #20 supersede) with four new fixes
discovered by running clawstor against itself + across the fabric:

* target_subdir_for: `dev`/`test` → `debug/`, `release`/`bench` →
  `release/`, custom passes through. Was silently skipping upload.

* rustc release via gossip: daemon probes `rustc --version` at start,
  publishes the release string as `clawstor.rustc.release`. PeerView
  carries it; `cluster-peer-status` prints it in a new column and
  emits a warning line when the fleet has mixed versions. Would have
  surfaced the tank/architect 1.96.1 vs 1.95.0 drift instantly.

* prewarm publishes fingerprint→blob ref downstream: `pin` now writes
  a companion tag `<name>.fingerprint` holding the fingerprint bytes.
  `prewarm` reads the companion, PutTag's it downstream, then
  PutRef(fp→blob) so a subsequent fingerprint-based `build` HITS.
  Without this, prewarm was almost useless for the runner path
  (build always missed even with matching source + rustc).

* streaming byte counters: BlobPutStream + BlobGetStream now record
  the transferred bytes via `record_blob_{put,get}_bytes`. Metric
  used to stay at 0 no matter how much you moved.

* capture determinism: replaced `tar::Builder::append_dir_all` (uses
  `read_dir`'s native order) with `append_dir_sorted` that walks the
  tree recursively and sorts by filename bytes at every level. Two
  byte-identical trees now produce byte-identical tars regardless of
  filesystem ordering.

+3 tests:
- target_subdir_matches_cargo_layout (from #20)
- fingerprint_companion_tag_uses_dotted_suffix
- capture_is_order_independent_of_filesystem_readdir (guard against
  the exact bug we saw in the field)

252 tests pass (+1 from Phase 5h's 251). Pre-existing macOS `du -sb`
failure unchanged.

Supersedes #20 (also included here). Ready for re-deploy to
tank + architect for the retest run.

claw-store/src/claw_cargo.rs

@@ -365,7 +365,9 @@ async fn cmd_status(args: PeerArgs) -> Result<()> {
 
 async fn cmd_prefetch(args: PrefetchArgs) -> Result<()> {
     let (workspace, resolved, fp) = setup_peer(&args.peer)?;
-    let target_dir = workspace.join("target").join(&resolved.profile);
+    let target_dir = workspace
+        .join("target")
+        .join(target_subdir_for(&resolved.profile));
 
     let (client, conn) = connect_peer(&resolved).await?;
 
@@ -485,7 +487,9 @@ async fn resolve_pin(
 
 async fn cmd_build(args: BuildArgs) -> Result<()> {
     let (workspace, resolved, fp) = setup_peer(&args.peer)?;
-    let target_dir = workspace.join("target").join(&resolved.profile);
+    let target_dir = workspace
+        .join("target")
+        .join(target_subdir_for(&resolved.profile));
     tracing::info!("fingerprint {}", fp);
 
     let (client, conn) = connect_peer(&resolved).await?;
@@ -618,15 +622,31 @@ async fn cmd_pin(args: PinArgs) -> Result<()> {
     // 2. Publish the tag → BlobId mapping.
     call_put_tag(&conn, &args.name, blob_id.as_bytes()).await?;
 
+    // Field finding 2026-07-12: also stash the fingerprint under a
+    // companion tag `<name>.fingerprint`. Downstream nodes fed by
+    // prewarm need to know the fingerprint to publish their own
+    // ref (`PutRef(fingerprint → blob)`) — otherwise a `build` on
+    // the same source misses even though the blob is present.
+    let companion = fingerprint_companion_tag(&args.name);
+    call_put_tag(&conn, &companion, fp.as_bytes()).await?;
+
     conn.close(quinn::VarInt::from_u32(0), b"done");
     client.shutdown().await;
 
     println!("pinned:      {}", args.name);
+    println!("companion:   {}", companion);
     println!("fingerprint: {}", fp);
     println!("blob:        {}", blob_id);
     Ok(())
 }
 
+/// Field finding 2026-07-12: companion tag suffix used by `pin` to
+/// stash the fingerprint alongside the blob-id mapping. Prewarm reads
+/// it to know what ref to publish downstream.
+fn fingerprint_companion_tag(name: &str) -> String {
+    format!("{name}.fingerprint")
+}
+
 async fn cmd_unpin(args: UnpinArgs) -> Result<()> {
     let workspace = args
         .peer
@@ -880,6 +900,23 @@ async fn cmd_prewarm(args: PrewarmArgs) -> Result<()> {
     // there would still miss.
     call_put_tag(&down_conn, &args.pin, blob_id.as_bytes()).await?;
 
+    // Field finding 2026-07-12: also publish the fingerprint → blob
+    // ref downstream when we can find the companion tag written by
+    // `pin`. Without this, a `build` on downstream MISSES on the
+    // matching fingerprint even though prewarm just put the blob
+    // there — the runner would rebuild from scratch and re-upload.
+    // Best-effort: absence of the companion tag (older pin) just
+    // means we skip this step.
+    let companion = fingerprint_companion_tag(&args.pin);
+    let fingerprint_published = match call_get_tag(&up_conn, &companion).await? {
+        Some(fp_bytes) => {
+            call_put_tag(&down_conn, &companion, &fp_bytes).await?;
+            call_put_ref(&down_conn, &fp_bytes, blob_id.as_bytes()).await?;
+            true
+        }
+        None => false,
+    };
+
     up_conn.close(quinn::VarInt::from_u32(0), b"done");
     down_conn.close(quinn::VarInt::from_u32(0), b"done");
     up_client.shutdown().await;
@@ -906,6 +943,14 @@ async fn cmd_prewarm(args: PrewarmArgs) -> Result<()> {
         "dedup save:  {} chunks",
         (stat.chunk_count as usize).saturating_sub(uploaded_chunks)
     );
+    println!(
+        "ref published: {}",
+        if fingerprint_published {
+            "yes (fingerprint→blob mapped downstream — build will HIT)"
+        } else {
+            "no (companion tag absent — build will still MISS on downstream)"
+        }
+    );
     println!("elapsed:     {:?}", transfer_elapsed);
     println!("────────────────────────────────────────────────────");
     Ok(())
@@ -928,3 +973,42 @@ fn run_cargo(
     let status = cmd.status().context("spawning cargo build")?;
     Ok(status)
 }
+
+/// Map a cargo profile name to the directory under `target/` cargo
+/// actually writes to. Cargo aliases `dev`/`test` → `debug/` and
+/// `release`/`bench` → `release/`; custom profiles get a dir of their
+/// own name. First discovered in the field 2026-07-12 — silent upload
+/// skip when the pilot ran with the default `profile = "dev"`.
+fn target_subdir_for(profile: &str) -> &str {
+    match profile {
+        "dev" | "test" => "debug",
+        "release" | "bench" => "release",
+        other => other,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn target_subdir_matches_cargo_layout() {
+        assert_eq!(target_subdir_for("dev"), "debug");
+        assert_eq!(target_subdir_for("test"), "debug");
+        assert_eq!(target_subdir_for("release"), "release");
+        assert_eq!(target_subdir_for("bench"), "release");
+        assert_eq!(target_subdir_for("prod"), "prod");
+        assert_eq!(target_subdir_for("hot-loop"), "hot-loop");
+    }
+
+    #[test]
+    fn fingerprint_companion_tag_uses_dotted_suffix() {
+        // Kept as a plain function so the shape is easy for a
+        // downstream consumer to reproduce without linking. Any change
+        // here must be coordinated with `cmd_prewarm`.
+        assert_eq!(
+            fingerprint_companion_tag("clawverse:main:latest"),
+            "clawverse:main:latest.fingerprint"
+        );
+    }
+}

claw-store/src/cluster/build_cache.rs

@@ -232,6 +232,77 @@ impl std::fmt::Display for Fingerprint {
 /// `target_dir` should be `<workspace>/target/<profile>` — the
 /// caller resolves the profile so this function doesn't have to
 /// know about cargo's directory layout beyond "the deps live here".
+/// Field finding 2026-07-12: walk `src` recursively in sorted order
+/// and append every regular file + directory to `tar` under
+/// `<archive_prefix>/<rel>`. Two calls on byte-identical trees produce
+/// byte-identical tar output (given `HeaderMode::Deterministic`), even
+/// across nodes whose `read_dir` returns entries in different orders.
+///
+/// Symlinks are appended as symlinks (the tar crate handles the header
+/// bookkeeping); anything else — sockets, fifos — is skipped.
+fn append_dir_sorted<W: std::io::Write>(
+    tar: &mut tar::Builder<W>,
+    archive_prefix: &str,
+    src: &Path,
+) -> Result<()> {
+    let mut stack: Vec<(PathBuf, String)> =
+        vec![(src.to_path_buf(), archive_prefix.to_string())];
+    while let Some((dir, archive_dir)) = stack.pop() {
+        let mut entries: Vec<_> = std::fs::read_dir(&dir)
+            .with_context(|| format!("reading {}", dir.display()))?
+            .filter_map(|e| e.ok())
+            .collect();
+        // Sort by filename bytes — stable across filesystems.
+        entries.sort_by(|a, b| a.file_name().cmp(&b.file_name()));
+        for entry in entries {
+            let ft = entry.file_type()?;
+            let name = entry.file_name();
+            let name_str = match name.to_str() {
+                Some(s) => s,
+                None => continue,
+            };
+            let archive_path = format!("{}/{}", archive_dir, name_str);
+            let full = entry.path();
+            if ft.is_dir() {
+                // Push for later processing; also emit the directory
+                // header so an empty dir survives the roundtrip.
+                let mut header = tar::Header::new_gnu();
+                header.set_entry_type(tar::EntryType::Directory);
+                header.set_size(0);
+                header.set_mode(0o755);
+                header.set_mtime(0);
+                header.set_cksum();
+                tar.append_data(
+                    &mut header,
+                    format!("{}/", archive_path),
+                    std::io::empty(),
+                )?;
+                stack.push((full, archive_path));
+            } else if ft.is_file() {
+                let mut f = std::fs::File::open(&full)
+                    .with_context(|| format!("opening {}", full.display()))?;
+                tar.append_file(&archive_path, &mut f)
+                    .with_context(|| format!("appending {}", full.display()))?;
+            } else if ft.is_symlink() {
+                let link_target = std::fs::read_link(&full)
+                    .with_context(|| format!("reading symlink {}", full.display()))?;
+                let mut header = tar::Header::new_gnu();
+                header.set_entry_type(tar::EntryType::Symlink);
+                header.set_size(0);
+                header.set_mode(0o777);
+                header.set_mtime(0);
+                header
+                    .set_link_name(&link_target)
+                    .context("setting symlink header link_name")?;
+                header.set_cksum();
+                tar.append_data(&mut header, &archive_path, std::io::empty())?;
+            }
+            // Other types (sockets, fifos) are skipped.
+        }
+    }
+    Ok(())
+}
+
 pub fn capture_target(target_dir: &Path) -> Result<Vec<u8>> {
     if !target_dir.is_dir() {
         bail!(
@@ -247,10 +318,21 @@ pub fn capture_target(target_dir: &Path) -> Result<Vec<u8>> {
         tar.mode(tar::HeaderMode::Deterministic);
         tar.follow_symlinks(false);
 
+        // Field finding 2026-07-12: `tar::Builder::append_dir_all` walks
+        // via `std::fs::read_dir`, which returns entries in
+        // filesystem-native order — non-deterministic across nodes even
+        // when contents are byte-identical. That drives blob-id drift
+        // between peers building the same source with the same rustc,
+        // which in turn caps cross-node dedup savings.
+        //
+        // Walk each subdir ourselves + sort by relative path before
+        // appending so two nodes building the same tree emit
+        // byte-identical tars (up to non-deterministic file contents
+        // like rustc debug-info paths, which live elsewhere).
         for sub in CAPTURED_SUBDIRS {
             let path = target_dir.join(sub);
             if path.is_dir() {
-                tar.append_dir_all(sub, &path)
+                append_dir_sorted(&mut tar, sub, &path)
                     .with_context(|| format!("archiving {}", path.display()))?;
             }
         }
@@ -531,6 +613,40 @@ mod tests {
         );
     }
 
+    #[test]
+    fn capture_is_order_independent_of_filesystem_readdir() {
+        // Field finding 2026-07-12: `append_dir_all` used
+        // `read_dir`'s native order, which differs across filesystems.
+        // Two byte-identical trees produced different tars whose only
+        // difference was entry order. Guard: create two trees whose
+        // files are the same but written in DIFFERENT orders (which
+        // biases readdir on many FS layouts), and require the
+        // captures to match. `append_dir_sorted` — the new walker —
+        // sorts by filename so the order at capture time is fixed.
+        let tmp = tempfile::TempDir::new().unwrap();
+        let a = tmp.path().join("A");
+        let b = tmp.path().join("B");
+        // Tree A: write a, b, c
+        write_file(&a, "deps/aaa.rlib", b"aaa content");
+        write_file(&a, "deps/bbb.rlib", b"bbb content");
+        write_file(&a, "deps/ccc.rlib", b"ccc content");
+        write_file(&a, ".fingerprint/aaa/xxx", b"fp-aaa");
+        write_file(&a, ".fingerprint/bbb/xxx", b"fp-bbb");
+        // Tree B: same files, reverse creation order
+        write_file(&b, "deps/ccc.rlib", b"ccc content");
+        write_file(&b, "deps/bbb.rlib", b"bbb content");
+        write_file(&b, "deps/aaa.rlib", b"aaa content");
+        write_file(&b, ".fingerprint/bbb/xxx", b"fp-bbb");
+        write_file(&b, ".fingerprint/aaa/xxx", b"fp-aaa");
+
+        let cap_a = capture_target(&a).unwrap();
+        let cap_b = capture_target(&b).unwrap();
+        assert_eq!(
+            cap_a, cap_b,
+            "captures must be byte-identical after sorted walk"
+        );
+    }
+
     #[test]
     fn capture_yields_identical_bytes_for_identical_input() {
         // With HeaderMode::Deterministic on the tar builder, two

claw-store/src/cluster/gossip.rs

@@ -61,6 +61,12 @@ pub mod keys {
     /// Phase 5i: cumulative bytes ingested into this node's blob store
     /// via `BlobPut`.
     pub const CACHE_BLOB_PUT_BYTES: &str = "clawstor.cache.blob_put.bytes";
+    /// Field finding 2026-07-12: `rustc --version --verbose` short form
+    /// — the "release" line only, e.g. `1.97.0`. Fingerprints depend on
+    /// the full verbose output, so a mismatch here is a strong hint
+    /// that two nodes will silo their caches. Peer-visible via
+    /// `PeerView.rustc_release` and `cluster-peer-status`.
+    pub const RUSTC_RELEASE: &str = "clawstor.rustc.release";
 }
 
 /// Cluster identifier — every node in the same fleet must agree on this
@@ -110,6 +116,11 @@ pub struct PeerView {
     pub cache_blob_get_bytes: Option<u64>,
     /// Phase 5i: cumulative bytes ingested into this node's blob store.
     pub cache_blob_put_bytes: Option<u64>,
+    /// Field finding 2026-07-12: peer's `rustc --version` release
+    /// string. `None` while the peer boots or when it can't invoke
+    /// rustc. Used to surface toolchain drift that would otherwise
+    /// silently silo caches.
+    pub rustc_release: Option<String>,
 }
 
 impl PeerView {
@@ -277,6 +288,13 @@ impl ClusterGossip {
         state.set(keys::CACHE_BLOB_PUT_BYTES, snap.blob_put_bytes.to_string());
     }
 
+    /// Field finding 2026-07-12: publish the local rustc release
+    /// string. Called at daemon startup so peers can flag mismatches
+    /// before wasting a build on a cache that will silo.
+    pub async fn set_rustc_release(&self, release: impl Into<String>) {
+        self.set(keys::RUSTC_RELEASE, release).await;
+    }
+
     /// Publish the list of warm-tier `org/repo` projects this node serves.
     /// Later phases use this to bias runner scheduling.
     pub async fn set_warm_projects<S: AsRef<str>>(&self, projects: &[S]) {
@@ -384,6 +402,7 @@ fn peer_view_from_state(id: &ChitchatId, state: &chitchat::NodeState, alive: boo
         cache_get_ref_misses: get_u64(state, keys::CACHE_GET_REF_MISSES),
         cache_blob_get_bytes: get_u64(state, keys::CACHE_BLOB_GET_BYTES),
         cache_blob_put_bytes: get_u64(state, keys::CACHE_BLOB_PUT_BYTES),
+        rustc_release: get_str(state, keys::RUSTC_RELEASE),
     }
 }
 
@@ -699,6 +718,7 @@ mod tests {
             cache_get_ref_misses: None,
             cache_blob_get_bytes: None,
             cache_blob_put_bytes: None,
+            rustc_release: None,
         };
         assert_eq!(base.cache_get_ref_hit_rate(), None, "no counters → None");
 
@@ -744,6 +764,7 @@ mod tests {
             cache_get_ref_misses: None,
             cache_blob_get_bytes: None,
             cache_blob_put_bytes: None,
+            rustc_release: None,
         };
         assert_eq!(base.hot_fill_ratio(), None, "no used → None");
 

claw-store/src/cluster/rpc.rs

@@ -729,6 +729,14 @@ async fn handle_blob_put_stream(
     };
     match store.put_stream(recv).await {
         Ok(id) => {
+            // Field finding 2026-07-12: the streaming variants had never
+            // been counted, so `clawstor_cache_blob_put_bytes_total`
+            // stayed at 0 even after multi-MB uploads. Look up the size
+            // via `stat` — cheap (single manifest read) and gives the
+            // authoritative post-store byte count.
+            if let Ok(Some(stat)) = store.stat(&id).await {
+                router.metrics.record_blob_put_bytes(stat.total_size);
+            }
             let mut reply = Vec::with_capacity(33);
             reply.push(STREAM_STATUS_OK);
             reply.extend_from_slice(id.as_bytes());
@@ -772,9 +780,14 @@ async fn handle_blob_get_stream(
         None => {
             send.write_all(&[ErrorCode::NotFound.as_byte()]).await?;
         }
-        Some(_) => {
+        Some(manifest) => {
             send.write_all(&[STREAM_STATUS_OK]).await?;
             store.stream_to(&id, &mut send).await?;
+            // Field finding 2026-07-12: streaming GETs weren't counted,
+            // leaving `blob_get_bytes_total` at 0. Record the manifest's
+            // authoritative total_size — we've committed to serving the
+            // whole thing by this point.
+            router.metrics.record_blob_get_bytes(manifest.total_size);
         }
     }
     send.finish()?;

claw-store/src/cluster/services.rs

@@ -116,6 +116,17 @@ impl ClusterServices {
         // Publish the static config value once. Used-bytes updates every tick.
         gossip.set_hot_max(hot_max_bytes).await;
 
+        // Field finding 2026-07-12: publish `rustc --version` so peers
+        // can flag toolchain drift before wasting a build on a cache
+        // that will silo. Best-effort — a node with no rustc on PATH
+        // simply doesn't advertise; peer-metrics prints "unknown".
+        if let Some(release) = detect_rustc_release() {
+            tracing::info!(rustc = %release, "publishing rustc release into gossip");
+            gossip.set_rustc_release(release).await;
+        } else {
+            tracing::info!("rustc not detected on PATH; skipping rustc.release gossip key");
+        }
+
         // Open the local blob store if a root path was supplied. Kept
         // outside the TLS branch: a node can serve blobs to callers
         // without RPC (via in-process API) or over RPC (once TLS is
@@ -380,6 +391,27 @@ fn dir_bytes_sync(root: &Path) -> u64 {
     total
 }
 
+/// Field finding 2026-07-12: probe `rustc --version` at startup so the
+/// daemon can advertise its toolchain over gossip. Best-effort; if
+/// rustc isn't on PATH we return `None` and skip the publish.
+///
+/// Returns the "release" component only — for `rustc 1.97.0 (...)`
+/// that's `1.97.0`. Matches what fingerprints care about most: a bump
+/// in the major/minor version guarantees a different fingerprint.
+fn detect_rustc_release() -> Option<String> {
+    let output = std::process::Command::new("rustc")
+        .arg("--version")
+        .output()
+        .ok()?;
+    if !output.status.success() {
+        return None;
+    }
+    let line = std::str::from_utf8(&output.stdout).ok()?.trim();
+    // Format: `rustc 1.97.0 (2d8144b78 2026-07-07)` — second whitespace
+    // token is the release.
+    line.split_whitespace().nth(1).map(|s| s.to_string())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

claw-store/src/main.rs

@@ -252,10 +252,19 @@ async fn cmd_cluster_peer_status(
         println!("  (no peers known)");
     } else {
         println!(
-            "  {:<20} {:<14} {:<8} {:<22} {:<20}",
+            "  {:<20} {:<14} {:<8} {:<22} {:<10} {:<20}",
-            "NAME", "ZONE", "STATE", "RPC LAN", "HOT USED / MAX"
+            "NAME", "ZONE", "STATE", "RPC LAN", "RUSTC", "HOT USED / MAX"
         );
-        println!("  {}", "-".repeat(90));
+        println!("  {}", "-".repeat(100));
+        // Field finding 2026-07-12: also render each peer's rustc
+        // release. Toolchain drift silently silos caches; showing it
+        // here means one glance surfaces the problem.
+        let local_rustc = status
+            .peers
+            .iter()
+            .filter_map(|p| p.rustc_release.clone())
+            .next()
+            .unwrap_or_default();
         for p in &status.peers {
             let state = if p.alive { "alive" } else { "dead" };
             let hot = match (p.hot_used_bytes, p.hot_max_bytes) {
@@ -263,17 +272,41 @@ async fn cmd_cluster_peer_status(
                 (Some(u), None) => format!("{u} / -"),
                 _ => "-".into(),
             };
+            let rustc = p.rustc_release.as_deref().unwrap_or("-");
+            let mismatch = !local_rustc.is_empty()
+                && !rustc.is_empty()
+                && rustc != local_rustc
+                && local_rustc != "-";
+            let rustc_col = if mismatch {
+                format!("{rustc}!")
+            } else {
+                rustc.to_string()
+            };
             println!(
-                "  {:<20} {:<14} {:<8} {:<22} {:<20}",
+                "  {:<20} {:<14} {:<8} {:<22} {:<10} {:<20}",
                 p.name,
                 p.zone,
                 state,
                 p.rpc_lan
                     .map(|a| a.to_string())
                     .unwrap_or_else(|| "-".into()),
+                rustc_col,
                 hot,
             );
         }
+        if status
+            .peers
+            .iter()
+            .filter_map(|p| p.rustc_release.as_deref())
+            .collect::<std::collections::HashSet<_>>()
+            .len()
+            > 1
+        {
+            println!();
+            println!(
+                "  ⚠  rustc release mismatch across peers → fingerprints will silo caches"
+            );
+        }
     }
 
     conn.close(quinn::VarInt::from_u32(0), b"done");