Phase 1c: QUIC transport with fleet-CA mTLS #3

Merged
osobh merged 1 commits from phase-1c-quic-transport into main 2026-07-12 04:55:14 +00:00
7 changed files with 947 additions and 22 deletions
Showing only changes of commit 4d93f955c5 - Show all commits
Generated
+309 -12
View File
@@ -62,7 +62,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [ dependencies = [
"windows-sys", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -73,7 +73,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [ dependencies = [
"anstyle", "anstyle",
"once_cell_polyfill", "once_cell_polyfill",
"windows-sys", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -172,6 +172,12 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "base64"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "2.13.0" version = "2.13.0"
@@ -208,6 +214,12 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]] [[package]]
name = "chacha20" name = "chacha20"
version = "0.10.1" version = "0.10.1"
@@ -303,6 +315,9 @@ dependencies = [
"clap", "clap",
"http-body-util", "http-body-util",
"libc", "libc",
"quinn",
"rcgen",
"rustls",
"serde", "serde",
"serde_json", "serde_json",
"sysinfo", "sysinfo",
@@ -362,6 +377,12 @@ version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "deranged"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
[[package]] [[package]]
name = "either" name = "either"
version = "1.16.0" version = "1.16.0"
@@ -381,7 +402,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [ dependencies = [
"libc", "libc",
"windows-sys", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -456,6 +477,19 @@ dependencies = [
"slab", "slab",
] ]
[[package]]
name = "getrandom"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"wasi",
"wasm-bindgen",
]
[[package]] [[package]]
name = "getrandom" name = "getrandom"
version = "0.4.2" version = "0.4.2"
@@ -463,11 +497,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"js-sys",
"libc", "libc",
"r-efi", "r-efi",
"rand_core", "rand_core",
"wasip2", "wasip2",
"wasip3", "wasip3",
"wasm-bindgen",
] ]
[[package]] [[package]]
@@ -651,7 +687,7 @@ version = "0.1.35"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
dependencies = [ dependencies = [
"getrandom", "getrandom 0.4.2",
"libc", "libc",
] ]
@@ -714,6 +750,12 @@ dependencies = [
"hashbrown 0.17.1", "hashbrown 0.17.1",
] ]
[[package]]
name = "lru-slab"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]] [[package]]
name = "matchers" name = "matchers"
version = "0.2.0" version = "0.2.0"
@@ -759,7 +801,7 @@ checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
dependencies = [ dependencies = [
"libc", "libc",
"wasi", "wasi",
"windows-sys", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -777,9 +819,15 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [ dependencies = [
"windows-sys", "windows-sys 0.61.2",
] ]
[[package]]
name = "num-conv"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
[[package]] [[package]]
name = "num-traits" name = "num-traits"
version = "0.2.19" version = "0.2.19"
@@ -824,6 +872,16 @@ dependencies = [
"windows-link", "windows-link",
] ]
[[package]]
name = "pem"
version = "3.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be"
dependencies = [
"base64",
"serde_core",
]
[[package]] [[package]]
name = "percent-encoding" name = "percent-encoding"
version = "2.3.2" version = "2.3.2"
@@ -842,6 +900,12 @@ version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
[[package]]
name = "powerfmt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]] [[package]]
name = "prettyplease" name = "prettyplease"
version = "0.2.37" version = "0.2.37"
@@ -861,6 +925,62 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "quinn"
version = "0.11.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8"
dependencies = [
"bytes",
"cfg_aliases",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash",
"rustls",
"socket2",
"thiserror",
"tokio",
"tracing",
"web-time",
]
[[package]]
name = "quinn-proto"
version = "0.11.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
dependencies = [
"bytes",
"getrandom 0.4.2",
"lru-slab",
"rand",
"rand_pcg",
"ring",
"rustc-hash",
"rustls",
"rustls-pki-types",
"slab",
"thiserror",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694"
dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2",
"tracing",
"windows-sys 0.61.2",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.45" version = "1.0.45"
@@ -883,7 +1003,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
dependencies = [ dependencies = [
"chacha20", "chacha20",
"getrandom", "getrandom 0.4.2",
"rand_core", "rand_core",
] ]
@@ -893,6 +1013,15 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
[[package]]
name = "rand_pcg"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
dependencies = [
"rand_core",
]
[[package]] [[package]]
name = "rayon" name = "rayon"
version = "1.12.0" version = "1.12.0"
@@ -913,6 +1042,19 @@ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]]
name = "rcgen"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2"
dependencies = [
"pem",
"ring",
"rustls-pki-types",
"time",
"yasna",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.5.18" version = "0.5.18"
@@ -939,6 +1081,26 @@ version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
[[package]]
name = "ring"
version = "0.17.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
dependencies = [
"cc",
"cfg-if",
"getrandom 0.2.17",
"libc",
"untrusted",
"windows-sys 0.52.0",
]
[[package]]
name = "rustc-hash"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "1.1.4" version = "1.1.4"
@@ -949,7 +1111,42 @@ dependencies = [
"errno", "errno",
"libc", "libc",
"linux-raw-sys", "linux-raw-sys",
"windows-sys", "windows-sys 0.61.2",
]
[[package]]
name = "rustls"
version = "0.23.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f"
dependencies = [
"once_cell",
"ring",
"rustls-pki-types",
"rustls-webpki",
"subtle",
"zeroize",
]
[[package]]
name = "rustls-pki-types"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
dependencies = [
"web-time",
"zeroize",
]
[[package]]
name = "rustls-webpki"
version = "0.103.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
dependencies = [
"ring",
"rustls-pki-types",
"untrusted",
] ]
[[package]] [[package]]
@@ -1095,7 +1292,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
dependencies = [ dependencies = [
"libc", "libc",
"windows-sys", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -1104,6 +1301,12 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "subtle"
version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.117" version = "2.0.117"
@@ -1143,10 +1346,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [ dependencies = [
"fastrand", "fastrand",
"getrandom", "getrandom 0.4.2",
"once_cell", "once_cell",
"rustix", "rustix",
"windows-sys", "windows-sys 0.61.2",
]
[[package]]
name = "thiserror"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
dependencies = [
"proc-macro2",
"quote",
"syn",
] ]
[[package]] [[package]]
@@ -1158,6 +1381,40 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "time"
version = "0.3.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50"
dependencies = [
"deranged",
"num-conv",
"powerfmt",
"serde_core",
"time-core",
]
[[package]]
name = "time-core"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
[[package]]
name = "tinyvec"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "1.52.3" version = "1.52.3"
@@ -1172,7 +1429,7 @@ dependencies = [
"signal-hook-registry", "signal-hook-registry",
"socket2", "socket2",
"tokio-macros", "tokio-macros",
"windows-sys", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -1385,6 +1642,12 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]]
name = "untrusted"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]] [[package]]
name = "utf8parse" name = "utf8parse"
version = "0.2.2" version = "0.2.2"
@@ -1500,6 +1763,16 @@ dependencies = [
"semver", "semver",
] ]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.3.9" version = "0.3.9"
@@ -1600,6 +1873,15 @@ dependencies = [
"windows-link", "windows-link",
] ]
[[package]]
name = "windows-sys"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets",
]
[[package]] [[package]]
name = "windows-sys" name = "windows-sys"
version = "0.61.2" version = "0.61.2"
@@ -1776,6 +2058,21 @@ dependencies = [
"wasmparser", "wasmparser",
] ]
[[package]]
name = "yasna"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd"
dependencies = [
"time",
]
[[package]]
name = "zeroize"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
[[package]] [[package]]
name = "zmij" name = "zmij"
version = "1.0.21" version = "1.0.21"
+11
View File
@@ -32,6 +32,17 @@ tempfile = "3"
# detection. Wraps in cluster/gossip.rs. UDP transport, keyed KV state per # detection. Wraps in cluster/gossip.rs. UDP transport, keyed KV state per
# node, seed_nodes bootstrap from [[cluster.peers]] config. # node, seed_nodes bootstrap from [[cluster.peers]] config.
chitchat = "0.11" chitchat = "0.11"
# v0.11 — QUIC transport for peer RPC (Phase 1c). UDP-based; runs on a
# separate port from chitchat gossip. TLS 1.3 by default; we wire mTLS
# against a fleet root CA in cluster/transport.rs.
quinn = { version = "0.11", default-features = false, features = ["runtime-tokio", "rustls-ring"] }
# v0.23 — TLS 1.3 provider driving quinn's crypto. Pinned to the ring
# provider so a single CryptoProvider is installed process-wide.
rustls = { version = "0.23", default-features = false, features = ["ring"] }
# v0.13 — X.509 cert generation for the fleet CA + per-node leaf certs.
# Used both by production bootstrap (writes PEM to /etc/claw-store/tls/)
# and by tests (in-memory ephemeral CA).
rcgen = { version = "0.13", features = ["pem"] }
[dev-dependencies] [dev-dependencies]
tempfile = "3" tempfile = "3"
+1
View File
@@ -15,6 +15,7 @@
//! propagate without a daemon restart. //! propagate without a daemon restart.
pub mod gossip; pub mod gossip;
pub mod transport;
use crate::config::PeerEntry; use crate::config::PeerEntry;
use anyhow::{bail, Result}; use anyhow::{bail, Result};
+28 -7
View File
@@ -171,11 +171,15 @@ impl ClusterGossip {
kv(keys::ZONE, &cluster.zone), kv(keys::ZONE, &cluster.zone),
kv(keys::UPTIME_UNIX, generation.to_string()), kv(keys::UPTIME_UNIX, generation.to_string()),
]; ];
if let Some(lan) = cluster.bind_lan { // Advertise the RPC address (QUIC), NOT the gossip address.
initial_kvs.push(kv(keys::RPC_ADDR_LAN, lan.to_string())); // Gossip lives on `bind_lan`; RPC lives on `bind_lan.port + 1`
// (or the explicit `bind_rpc_lan` override) so both protocols
// — both UDP-based — don't collide.
if let Some(rpc_lan) = cluster.rpc_lan() {
initial_kvs.push(kv(keys::RPC_ADDR_LAN, rpc_lan.to_string()));
} }
if let Some(ts) = cluster.bind_tailscale { if let Some(rpc_ts) = cluster.rpc_tailscale() {
initial_kvs.push(kv(keys::RPC_ADDR_TAILSCALE, ts.to_string())); initial_kvs.push(kv(keys::RPC_ADDR_TAILSCALE, rpc_ts.to_string()));
} }
let config = ChitchatConfig { let config = ChitchatConfig {
@@ -376,6 +380,8 @@ mod tests {
bind_lan: Some(loopback(port)), bind_lan: Some(loopback(port)),
bind_tailscale: None, bind_tailscale: None,
peers: vec![], peers: vec![],
bind_rpc_lan: None,
bind_rpc_tailscale: None,
}; };
let g = ClusterGossip::bootstrap(&cfg, "solo").await.unwrap(); let g = ClusterGossip::bootstrap(&cfg, "solo").await.unwrap();
let id = g.self_chitchat_id().await; let id = g.self_chitchat_id().await;
@@ -391,6 +397,8 @@ mod tests {
bind_lan: Some(loopback(next_lan_port())), bind_lan: Some(loopback(next_lan_port())),
bind_tailscale: None, bind_tailscale: None,
peers: vec![], peers: vec![],
bind_rpc_lan: None,
bind_rpc_tailscale: None,
}; };
let err = ClusterGossip::bootstrap(&cfg, "") let err = ClusterGossip::bootstrap(&cfg, "")
.await .await
@@ -406,6 +414,8 @@ mod tests {
bind_lan: None, bind_lan: None,
bind_tailscale: None, bind_tailscale: None,
peers: vec![], peers: vec![],
bind_rpc_lan: None,
bind_rpc_tailscale: None,
}; };
// ClusterConfig::validate rejects this first — that's what we want: // ClusterConfig::validate rejects this first — that's what we want:
// the daemon should refuse to bootstrap gossip on a malformed config. // the daemon should refuse to bootstrap gossip on a malformed config.
@@ -429,6 +439,8 @@ mod tests {
bind_lan: Some(addr_a), bind_lan: Some(addr_a),
bind_tailscale: None, bind_tailscale: None,
peers: vec![], peers: vec![],
bind_rpc_lan: None,
bind_rpc_tailscale: None,
}; };
// Node B: uses A as seed. // Node B: uses A as seed.
let cfg_b = ClusterConfig { let cfg_b = ClusterConfig {
@@ -441,6 +453,8 @@ mod tests {
lan_addr: Some(addr_a), lan_addr: Some(addr_a),
tailscale_addr: None, tailscale_addr: None,
}], }],
bind_rpc_lan: None,
bind_rpc_tailscale: None,
}; };
let gossip_a = ClusterGossip::bootstrap(&cfg_a, "a").await.unwrap(); let gossip_a = ClusterGossip::bootstrap(&cfg_a, "a").await.unwrap();
@@ -464,10 +478,15 @@ mod tests {
wait_until_peer_alive(&gossip_b, "a", Duration::from_secs(10)).await; wait_until_peer_alive(&gossip_b, "a", Duration::from_secs(10)).await;
assert!(b_sees_a_alive, "B should see A alive within 10s"); assert!(b_sees_a_alive, "B should see A alive within 10s");
// Verify the state B sees for A matches what A published. // Verify the state B sees for A matches what A published. RPC
// address is gossip port + 1 (see rpc_lan()) — B must see that.
let b_view_of_a = gossip_b.peer("a").await.expect("B has A"); let b_view_of_a = gossip_b.peer("a").await.expect("B has A");
assert_eq!(b_view_of_a.zone, "fabric-10g"); assert_eq!(b_view_of_a.zone, "fabric-10g");
assert_eq!(b_view_of_a.rpc_lan, Some(addr_a)); assert_eq!(
b_view_of_a.rpc_lan,
Some(loopback(port_a + 1)),
"advertised RPC addr = gossip port + 1"
);
assert!(b_view_of_a.alive); assert!(b_view_of_a.alive);
assert_eq!(b_view_of_a.hot_used_bytes, Some(1024)); assert_eq!(b_view_of_a.hot_used_bytes, Some(1024));
assert_eq!(b_view_of_a.hot_max_bytes, Some(1_000_000)); assert_eq!(b_view_of_a.hot_max_bytes, Some(1_000_000));
@@ -483,7 +502,7 @@ mod tests {
// And what A sees for B — zone should be lan-1g. // And what A sees for B — zone should be lan-1g.
let a_view_of_b = gossip_a.peer("b").await.expect("A has B"); let a_view_of_b = gossip_a.peer("b").await.expect("A has B");
assert_eq!(a_view_of_b.zone, "lan-1g"); assert_eq!(a_view_of_b.zone, "lan-1g");
assert_eq!(a_view_of_b.rpc_lan, Some(addr_b)); assert_eq!(a_view_of_b.rpc_lan, Some(loopback(port_b + 1)));
// Zone filtering. // Zone filtering.
let a_fabric = gossip_a.peers_in_zone("fabric-10g").await; let a_fabric = gossip_a.peers_in_zone("fabric-10g").await;
@@ -504,6 +523,8 @@ mod tests {
bind_lan: Some(loopback(port)), bind_lan: Some(loopback(port)),
bind_tailscale: None, bind_tailscale: None,
peers: vec![], peers: vec![],
bind_rpc_lan: None,
bind_rpc_tailscale: None,
}; };
let g = ClusterGossip::bootstrap(&cfg, "solo").await.unwrap(); let g = ClusterGossip::bootstrap(&cfg, "solo").await.unwrap();
// Solo cluster — peers() must never include self. // Solo cluster — peers() must never include self.
+485
View File
@@ -0,0 +1,485 @@
//! QUIC transport for peer RPC with fleet-CA mTLS.
//!
//! Runs on a UDP port distinct from chitchat gossip (see
//! [`ClusterConfig::rpc_lan`](crate::config::ClusterConfig::rpc_lan) —
//! defaults to gossip port + 1) so the two UDP-based protocols don't
//! collide.
//!
//! # Trust model
//!
//! A single fleet root CA signs every node's leaf certificate. Each
//! node loads its own cert + private key + the CA public cert into a
//! [`NodeIdentity`]. Both server and client verifiers require peer
//! certs to chain to the fleet CA — that's mTLS. A node without a
//! signed cert cannot join RPC.
//!
//! # Test model
//!
//! The [`NodeIdentity::generate_test_pair`] helper cuts an ephemeral
//! CA + two leaf certs in-memory for two named nodes. Both nodes share
//! the CA (so they trust each other) but have distinct leaves. This is
//! the same code path used in production; there is no test-only side
//! door in the transport itself.
//!
//! # ALPN
//!
//! The single protocol identifier `clawstor-rpc/1` is offered on every
//! connection; peers advertising anything else are rejected during the
//! TLS handshake.
use anyhow::{bail, Context, Result};
use quinn::{ClientConfig, Endpoint, ServerConfig, VarInt};
use rustls::pki_types::{CertificateDer, PrivateKeyDer};
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;
/// ALPN identifier all fleet RPC endpoints advertise + require.
pub const CLAWSTOR_RPC_ALPN: &[u8] = b"clawstor-rpc/1";
/// Idle timeout on connection — if no data for this long the connection
/// dies. Short enough to notice partitions, long enough to survive a
/// paused laptop.
const IDLE_TIMEOUT: Duration = Duration::from_secs(30);
/// Cap on any single RPC message payload. Ping/pong is tiny; other RPCs
/// stream larger payloads via streams-of-many-messages. Prevents an
/// adversary from allocating unbounded memory on a single message.
const MAX_MESSAGE_BYTES: usize = 16 * 1024;
/// Cryptographic identity a node presents on both sides of the RPC
/// endpoint. Fully specifies the mTLS setup: our own cert chain + key,
/// plus the CA cert we require the peer's cert to chain to.
///
/// [`PrivateKeyDer`] is not `Clone`, so `NodeIdentity` isn't derive-Clone
/// either. Use [`NodeIdentity::clone_id`] when a second owner is required.
#[derive(Debug)]
pub struct NodeIdentity {
/// This node's certificate chain (leaf + optional intermediates).
pub cert_chain: Vec<CertificateDer<'static>>,
/// This node's private key matching the leaf cert above.
pub key: PrivateKeyDer<'static>,
/// The fleet root CA. Both server verifier and client verifier
/// require the peer's chain to end at this cert.
pub trusted_ca: CertificateDer<'static>,
}
impl NodeIdentity {
/// Build a fresh fleet CA and a pair of signed leaf certs named
/// after `node_a` and `node_b`. Returns the two independent
/// identities — each carries its own leaf but shares the CA so
/// they trust each other's certs.
///
/// Used by tests + as a starting point for the production
/// bootstrap flow (write PEMs to `/etc/claw-store/tls/`).
pub fn generate_test_pair(node_a: &str, node_b: &str) -> Result<(Self, Self)> {
let ca_key = rcgen::KeyPair::generate().context("generating CA key")?;
let mut ca_params = rcgen::CertificateParams::new(vec![])
.context("building CA params")?;
ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained);
ca_params
.distinguished_name
.push(rcgen::DnType::CommonName, "clawstor fleet CA");
let ca_cert = ca_params
.self_signed(&ca_key)
.context("self-signing CA cert")?;
let ca_der = ca_cert.der().clone();
let id_a = build_leaf(node_a, &ca_cert, &ca_key, ca_der.clone())?;
let id_b = build_leaf(node_b, &ca_cert, &ca_key, ca_der)?;
Ok((id_a, id_b))
}
}
/// Sign a single leaf cert for `name` under the given CA and package it
/// with the CA cert into a [`NodeIdentity`].
fn build_leaf(
name: &str,
ca_cert: &rcgen::Certificate,
ca_key: &rcgen::KeyPair,
ca_der: CertificateDer<'static>,
) -> Result<NodeIdentity> {
let leaf_key = rcgen::KeyPair::generate().context("generating leaf key")?;
// Include the node name as a DNS SAN so a client connecting with
// server_name=name passes rustls's cert-name verification.
let mut leaf_params = rcgen::CertificateParams::new(vec![name.to_string()])
.context("building leaf params")?;
leaf_params
.distinguished_name
.push(rcgen::DnType::CommonName, name);
let leaf_cert = leaf_params
.signed_by(&leaf_key, ca_cert, ca_key)
.context("signing leaf cert with CA")?;
let leaf_der = leaf_cert.der().clone();
// rustls wants the private key in DER form.
let key_der = PrivateKeyDer::try_from(leaf_key.serialize_der())
.map_err(|e| anyhow::anyhow!("converting leaf key to rustls form: {e}"))?;
Ok(NodeIdentity {
cert_chain: vec![leaf_der],
key: key_der,
trusted_ca: ca_der,
})
}
/// A running QUIC RPC server. Owns the quinn endpoint; drop stops it.
pub struct QuicServer {
endpoint: Endpoint,
}
impl QuicServer {
/// Bind a QUIC listener on `bind` with mTLS enforced against the
/// fleet CA in `identity`. `bind` accepts port 0 to let the OS
/// assign one (useful in tests).
pub fn bind(bind: SocketAddr, identity: NodeIdentity) -> Result<Self> {
install_default_crypto_provider();
let server_crypto = build_server_crypto(&identity)?;
let quic_crypto = quinn::crypto::rustls::QuicServerConfig::try_from(server_crypto)
.context("wrapping rustls ServerConfig for quinn")?;
let mut server_config = ServerConfig::with_crypto(Arc::new(quic_crypto));
Arc::get_mut(&mut server_config.transport)
.expect("fresh transport config is unique")
.max_concurrent_uni_streams(0u8.into())
.max_idle_timeout(Some(
VarInt::from_u64(IDLE_TIMEOUT.as_millis() as u64)
.expect("idle timeout fits u64")
.into(),
));
let endpoint =
Endpoint::server(server_config, bind).context("binding quinn server endpoint")?;
Ok(Self { endpoint })
}
/// The actual bound socket address (resolves port 0 to the assigned port).
pub fn local_addr(&self) -> Result<SocketAddr> {
self.endpoint.local_addr().map_err(Into::into)
}
/// Accept the next incoming connection. Returns `None` when the
/// endpoint is closed. The returned [`quinn::Connection`] is
/// already handshaked; call `ping_handler_loop` (or any other
/// server-side handler) to serve requests on it.
pub async fn accept(&self) -> Option<Result<quinn::Connection>> {
let incoming = self.endpoint.accept().await?;
// Two failure modes matter here:
// 1. `incoming.accept()` returns Err → convert to a Result inside
// the returned Option so callers see the accept failure.
// 2. The Connecting future fails → same shape.
let connecting = match incoming.accept() {
Ok(c) => c,
Err(e) => return Some(Err(anyhow::Error::from(e))),
};
Some(connecting.await.map_err(anyhow::Error::from))
}
/// Graceful shutdown.
pub async fn shutdown(&self) {
self.endpoint
.close(VarInt::from_u32(0), b"server shutdown");
self.endpoint.wait_idle().await;
}
}
/// A QUIC client for opening outbound peer connections.
pub struct QuicClient {
endpoint: Endpoint,
client_config: ClientConfig,
}
impl QuicClient {
/// Build a client bound to `bind_addr` (usually `0.0.0.0:0` — OS
/// picks the port) that verifies peer certs against the fleet CA
/// carried in `identity`, and presents `identity`'s leaf on mTLS
/// challenge.
pub fn new(bind_addr: SocketAddr, identity: NodeIdentity) -> Result<Self> {
install_default_crypto_provider();
let client_crypto = build_client_crypto(&identity)?;
let quic_crypto = quinn::crypto::rustls::QuicClientConfig::try_from(client_crypto)
.context("wrapping rustls ClientConfig for quinn")?;
let client_config = ClientConfig::new(Arc::new(quic_crypto));
let endpoint = Endpoint::client(bind_addr).context("binding quinn client endpoint")?;
Ok(Self {
endpoint,
client_config,
})
}
/// Connect to `addr` and validate the peer cert's SAN matches
/// `expected_server_name`. Returns the completed connection.
pub async fn connect(
&self,
addr: SocketAddr,
expected_server_name: &str,
) -> Result<quinn::Connection> {
let connecting = self
.endpoint
.connect_with(self.client_config.clone(), addr, expected_server_name)
.with_context(|| format!("dialing {addr} for name {expected_server_name}"))?;
connecting
.await
.with_context(|| format!("completing handshake with {addr}"))
}
/// Graceful shutdown.
pub async fn shutdown(&self) {
self.endpoint
.close(VarInt::from_u32(0), b"client shutdown");
self.endpoint.wait_idle().await;
}
}
/// Open a bidi stream on `conn`, send `payload`, read back the peer's
/// response (bounded by [`MAX_MESSAGE_BYTES`]). This is the client
/// side of the ping RPC.
pub async fn ping(conn: &quinn::Connection, payload: &[u8]) -> Result<Vec<u8>> {
if payload.len() > MAX_MESSAGE_BYTES {
bail!(
"ping payload {} bytes exceeds cap {}",
payload.len(),
MAX_MESSAGE_BYTES
);
}
let (mut send, mut recv) = conn
.open_bi()
.await
.context("opening bidi stream for ping")?;
send.write_all(payload).await.context("writing ping payload")?;
send.finish().context("finishing ping send stream")?;
let response = recv
.read_to_end(MAX_MESSAGE_BYTES)
.await
.context("reading ping response")?;
Ok(response)
}
/// Server side of the ping RPC. Loops accepting bidi streams on `conn`
/// and echoing every payload back prefixed with `b"pong:"`. Runs until
/// the connection closes.
pub async fn ping_handler_loop(conn: quinn::Connection) -> Result<()> {
loop {
let (mut send, mut recv) = match conn.accept_bi().await {
Ok(pair) => pair,
Err(quinn::ConnectionError::ApplicationClosed(_))
| Err(quinn::ConnectionError::ConnectionClosed(_))
| Err(quinn::ConnectionError::LocallyClosed) => return Ok(()),
Err(e) => return Err(anyhow::Error::from(e)),
};
let request = recv
.read_to_end(MAX_MESSAGE_BYTES)
.await
.context("reading ping request")?;
let mut response = b"pong:".to_vec();
response.extend_from_slice(&request);
send.write_all(&response)
.await
.context("writing pong response")?;
send.finish().context("finishing pong send stream")?;
}
}
/// Build the server-side rustls config: require client certs, chain to
/// our fleet CA, present our own leaf on handshake.
fn build_server_crypto(identity: &NodeIdentity) -> Result<rustls::ServerConfig> {
let mut ca_store = rustls::RootCertStore::empty();
ca_store
.add(identity.trusted_ca.clone())
.context("installing fleet CA in server root store")?;
let client_verifier =
rustls::server::WebPkiClientVerifier::builder(Arc::new(ca_store))
.build()
.context("building mTLS client verifier")?;
let mut cfg = rustls::ServerConfig::builder()
.with_client_cert_verifier(client_verifier)
.with_single_cert(identity.cert_chain.clone(), identity.key.clone_key())
.context("installing server leaf cert + key")?;
cfg.alpn_protocols = vec![CLAWSTOR_RPC_ALPN.to_vec()];
Ok(cfg)
}
/// Build the client-side rustls config: verify server cert chains to
/// our fleet CA, present our own leaf when the server challenges.
fn build_client_crypto(identity: &NodeIdentity) -> Result<rustls::ClientConfig> {
let mut ca_store = rustls::RootCertStore::empty();
ca_store
.add(identity.trusted_ca.clone())
.context("installing fleet CA in client root store")?;
let mut cfg = rustls::ClientConfig::builder()
.with_root_certificates(ca_store)
.with_client_auth_cert(identity.cert_chain.clone(), identity.key.clone_key())
.context("installing client leaf cert + key")?;
cfg.alpn_protocols = vec![CLAWSTOR_RPC_ALPN.to_vec()];
Ok(cfg)
}
/// Install the ring-backed CryptoProvider once per process. Called from
/// both server and client bootstrap. Idempotent — ignores the "already
/// installed" error so multiple endpoints in the same process (or tests)
/// coexist.
fn install_default_crypto_provider() {
// `install_default` returns Err if a provider is already set — that's
// fine; we just want SOME provider present when rustls builders run.
let _ = rustls::crypto::ring::default_provider().install_default();
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicU16, Ordering};
/// Port allocator separate from the gossip tests (which use 41000+)
/// so parallel test execution never conflicts.
static NEXT_PORT: AtomicU16 = AtomicU16::new(42001);
fn next_port() -> u16 {
NEXT_PORT.fetch_add(1, Ordering::Relaxed)
}
fn loopback(port: u16) -> SocketAddr {
format!("127.0.0.1:{port}").parse().unwrap()
}
fn wildcard(port: u16) -> SocketAddr {
format!("0.0.0.0:{port}").parse().unwrap()
}
#[test]
fn generate_test_pair_produces_two_distinct_leaves_sharing_a_ca() {
let (a, b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
assert_ne!(
a.cert_chain[0], b.cert_chain[0],
"each node must have a distinct leaf"
);
assert_eq!(
a.trusted_ca, b.trusted_ca,
"both nodes must trust the same CA to talk"
);
assert!(!a.cert_chain.is_empty());
assert!(!b.cert_chain.is_empty());
}
#[tokio::test]
async fn ping_pong_between_two_mtls_peers() {
let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
// B binds a server on an OS-assigned port.
let server = QuicServer::bind(loopback(0), id_b).unwrap();
let server_addr = server.local_addr().unwrap();
// Spawn the accept loop: for every incoming connection, serve
// the ping RPC.
let accept_task = tokio::spawn(async move {
if let Some(res) = server.accept().await {
let conn = res.expect("accept succeeded");
let _ = ping_handler_loop(conn).await;
}
server.shutdown().await;
});
// A connects as a client and calls ping.
let client = QuicClient::new(loopback(0), id_a).unwrap();
let conn = client
.connect(server_addr, "b")
.await
.expect("A→B connect");
let response = ping(&conn, b"hello").await.expect("ping");
assert_eq!(response, b"pong:hello");
// Cleanup.
conn.close(VarInt::from_u32(0), b"done");
client.shutdown().await;
// Give the accept task a moment to finish, then abort to be safe.
tokio::time::sleep(Duration::from_millis(50)).await;
accept_task.abort();
}
#[tokio::test]
async fn client_rejects_peer_with_wrong_ca() {
// Two CAs, A and X. A pair (id_a, id_b) share CA_A; a rogue id_x
// has its own CA_X. When A tries to talk to X, TLS handshake
// must fail — X's leaf doesn't chain to CA_A.
let (id_a, _id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
let (_id_x_peer, id_x) = NodeIdentity::generate_test_pair("x-peer", "x").unwrap();
let server = QuicServer::bind(loopback(0), id_x).unwrap();
let server_addr = server.local_addr().unwrap();
let _accept_task = tokio::spawn(async move {
// Server will fail the handshake because the client's cert
// doesn't chain to CA_X. We just want to be here to accept
// the SYN; the handshake failure is what the client asserts.
let _ = server.accept().await;
});
let client = QuicClient::new(loopback(0), id_a).unwrap();
let err = client
.connect(server_addr, "x")
.await
.err()
.expect("connect must fail — wrong CA");
let msg = format!("{err:#}");
// The exact error text varies by rustls version; look for the
// token that always appears in a chain-verification failure.
assert!(
msg.to_lowercase().contains("certificate")
|| msg.to_lowercase().contains("verify")
|| msg.to_lowercase().contains("closed")
|| msg.to_lowercase().contains("handshake"),
"unexpected error text: {msg}"
);
}
#[tokio::test]
async fn client_rejects_wrong_server_name() {
// A and B share a CA; B's cert has SAN=["b"]. A connects
// asking for server_name="c" — rustls must reject on SAN
// mismatch even though the cert chain is valid.
let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
let server = QuicServer::bind(loopback(0), id_b).unwrap();
let server_addr = server.local_addr().unwrap();
let _accept_task = tokio::spawn(async move {
let _ = server.accept().await;
});
let client = QuicClient::new(loopback(0), id_a).unwrap();
let err = client
.connect(server_addr, "c")
.await
.err()
.expect("connect must fail — SAN mismatch");
let msg = format!("{err:#}").to_lowercase();
assert!(
msg.contains("name") || msg.contains("certificate") || msg.contains("handshake"),
"unexpected error text: {msg}"
);
}
#[tokio::test]
async fn server_binds_wildcard_and_reports_concrete_local_addr() {
let (_id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
let server = QuicServer::bind(wildcard(0), id_b).unwrap();
let addr = server.local_addr().unwrap();
assert_ne!(addr.port(), 0, "OS must assign a real port");
server.shutdown().await;
}
#[tokio::test]
async fn ping_rejects_oversize_payload() {
let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
let server = QuicServer::bind(loopback(0), id_b).unwrap();
let server_addr = server.local_addr().unwrap();
let _accept = tokio::spawn(async move {
if let Some(Ok(conn)) = server.accept().await {
let _ = ping_handler_loop(conn).await;
}
});
let client = QuicClient::new(loopback(0), id_a).unwrap();
let conn = client.connect(server_addr, "b").await.unwrap();
let oversized = vec![0u8; MAX_MESSAGE_BYTES + 1];
let err = ping(&conn, &oversized)
.await
.err()
.expect("must reject oversize payload before send");
assert!(err.to_string().contains("exceeds cap"));
conn.close(VarInt::from_u32(0), b"done");
client.shutdown().await;
}
}
+55 -3
View File
@@ -93,22 +93,39 @@ impl PeerEntry {
/// Cluster membership configuration. Optional at the top level so existing /// Cluster membership configuration. Optional at the top level so existing
/// single-node deployments (pre-v2) keep loading. Once present, describes the /// single-node deployments (pre-v2) keep loading. Once present, describes the
/// local node's zone + bind addresses, and enumerates known peers. /// local node's zone + bind addresses, and enumerates known peers.
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Default)]
pub struct ClusterConfig { pub struct ClusterConfig {
/// This node's zone tag. /// This node's zone tag.
pub zone: String, pub zone: String,
/// LAN listen socket (typically `0.0.0.0:7701`). Omit on roaming nodes. /// LAN listen socket for gossip (typically `0.0.0.0:7701`). Chitchat runs
/// on this UDP port. Omit on roaming nodes.
#[serde(default)] #[serde(default)]
pub bind_lan: Option<SocketAddr>, pub bind_lan: Option<SocketAddr>,
/// Tailscale listen socket (Tailscale IP + port). Omit on strictly-LAN nodes. /// Tailscale listen socket for gossip. Omit on strictly-LAN nodes.
#[serde(default)] #[serde(default)]
pub bind_tailscale: Option<SocketAddr>, pub bind_tailscale: Option<SocketAddr>,
/// LAN listen socket for RPC (QUIC). Defaults to `bind_lan.port + 1`
/// so gossip and RPC don't collide on the same UDP endpoint.
#[serde(default)]
pub bind_rpc_lan: Option<SocketAddr>,
/// Tailscale listen socket for RPC (QUIC). Defaults to
/// `bind_tailscale.port + 1`.
#[serde(default)]
pub bind_rpc_tailscale: Option<SocketAddr>,
/// Static seed list of peers. Runtime membership (Phase 1b) will extend this /// Static seed list of peers. Runtime membership (Phase 1b) will extend this
/// via gossip; the config list bootstraps discovery. /// via gossip; the config list bootstraps discovery.
#[serde(default)] #[serde(default)]
pub peers: Vec<PeerEntry>, pub peers: Vec<PeerEntry>,
} }
/// Compute the default RPC address for a gossip address: same IP, port + 1.
/// Used to derive `bind_rpc_lan` / `bind_rpc_tailscale` when the operator
/// hasn't set them explicitly.
fn default_rpc_addr(gossip: SocketAddr) -> Option<SocketAddr> {
let port = gossip.port().checked_add(1)?;
Some(SocketAddr::new(gossip.ip(), port))
}
impl ClusterConfig { impl ClusterConfig {
/// Sanity check the cluster config: at least one bind address, no duplicate /// Sanity check the cluster config: at least one bind address, no duplicate
/// peer names, every peer has at least one address. /// peer names, every peer has at least one address.
@@ -133,6 +150,37 @@ impl ClusterConfig {
pub fn peer(&self, name: &str) -> Option<&PeerEntry> { pub fn peer(&self, name: &str) -> Option<&PeerEntry> {
self.peers.iter().find(|p| p.name == name) self.peers.iter().find(|p| p.name == name)
} }
/// The LAN address this node listens on for RPC (QUIC). Prefers the
/// explicit `bind_rpc_lan` override; otherwise derives from `bind_lan`
/// with port + 1.
pub fn rpc_lan(&self) -> Option<SocketAddr> {
self.bind_rpc_lan.or_else(|| self.bind_lan.and_then(default_rpc_addr))
}
/// The Tailscale address this node listens on for RPC (QUIC).
pub fn rpc_tailscale(&self) -> Option<SocketAddr> {
self.bind_rpc_tailscale
.or_else(|| self.bind_tailscale.and_then(default_rpc_addr))
}
}
impl PeerEntry {
/// The peer's LAN RPC address, if it has one.
///
/// This is derived from the peer's `lan_addr` (which is their gossip
/// address in the config, matching how they identify themselves). RPC
/// runs on gossip port + 1 by convention. Gossip-based discovery in
/// Phase 1b will supplant this once a peer has broadcast its own
/// [`gossip::keys::RPC_ADDR_LAN`](crate::cluster::gossip::keys).
pub fn rpc_lan(&self) -> Option<SocketAddr> {
self.lan_addr.and_then(default_rpc_addr)
}
/// The peer's Tailscale RPC address.
pub fn rpc_tailscale(&self) -> Option<SocketAddr> {
self.tailscale_addr.and_then(default_rpc_addr)
}
} }
#[derive(Debug, Clone, Deserialize, Serialize)] #[derive(Debug, Clone, Deserialize, Serialize)]
@@ -323,6 +371,8 @@ tailscale_addr = "100.64.1.5:7701"
bind_lan: None, bind_lan: None,
bind_tailscale: None, bind_tailscale: None,
peers: vec![], peers: vec![],
bind_rpc_lan: None,
bind_rpc_tailscale: None,
}; };
let err = cluster.validate().unwrap_err().to_string(); let err = cluster.validate().unwrap_err().to_string();
assert!( assert!(
@@ -351,6 +401,8 @@ tailscale_addr = "100.64.1.5:7701"
tailscale_addr: None, tailscale_addr: None,
}, },
], ],
bind_rpc_lan: None,
bind_rpc_tailscale: None,
}; };
let err = cluster.validate().unwrap_err().to_string(); let err = cluster.validate().unwrap_err().to_string();
assert!( assert!(
+58
View File
@@ -16,6 +16,7 @@ use anyhow::{bail, Context, Result};
use clap::{Parser, Subcommand}; use clap::{Parser, Subcommand};
use config::Config; use config::Config;
use manifest::{Manifest, Project}; use manifest::{Manifest, Project};
use std::net::SocketAddr;
use std::path::PathBuf; use std::path::PathBuf;
use zfs::{SystemZfs, ZfsOps}; use zfs::{SystemZfs, ZfsOps};
@@ -83,6 +84,25 @@ enum Cmd {
#[arg(long, default_value = "3")] #[arg(long, default_value = "3")]
wait_secs: u64, wait_secs: u64,
}, },
/// Ping a peer over the QUIC RPC layer with an ephemeral fleet CA.
/// Demonstrates the mTLS handshake end-to-end against a live peer.
/// The `--rpc-addr` must match the peer's advertised RPC endpoint
/// (gossip port + 1 by default).
ClusterPing {
/// This node's name — used as the SAN in the ephemeral leaf cert.
#[arg(long)]
name: String,
/// Peer's node name — used as the expected server name (SAN) on
/// the incoming cert. Must match the peer's own `--name`.
#[arg(long)]
peer: String,
/// Peer's RPC socket. Typically gossip_port + 1.
#[arg(long)]
rpc_addr: SocketAddr,
/// Payload to send. Echoed back with a "pong:" prefix.
#[arg(long, default_value = "hello")]
payload: String,
},
} }
#[tokio::main] #[tokio::main]
@@ -119,10 +139,48 @@ async fn main() -> Result<()> {
Cmd::Unpin { project } => cmd_set_pin(&manifest_path, &project, false)?, Cmd::Unpin { project } => cmd_set_pin(&manifest_path, &project, false)?,
Cmd::ClusterProbe { peer } => cmd_cluster_probe(&cfg, &peer).await?, Cmd::ClusterProbe { peer } => cmd_cluster_probe(&cfg, &peer).await?,
Cmd::ClusterStatus { name, wait_secs } => cmd_cluster_status(&cfg, &name, wait_secs).await?, Cmd::ClusterStatus { name, wait_secs } => cmd_cluster_status(&cfg, &name, wait_secs).await?,
Cmd::ClusterPing {
name,
peer,
rpc_addr,
payload,
} => cmd_cluster_ping(&name, &peer, rpc_addr, &payload).await?,
} }
Ok(()) Ok(())
} }
// ── cluster ping ─────────────────────────────────────────────────────────────
/// Round-trip a `ping` payload to `peer` over the QUIC RPC transport
/// using an ephemeral fleet CA. Both this node and the peer must have
/// invoked `cluster-ping` with a shared CA — the current implementation
/// generates one CA per invocation via `NodeIdentity::generate_test_pair`,
/// meaning peers can only talk to each other when they share the same
/// invocation (typical dev-loopback usage). Production identity loading
/// from persistent PEM files lands in Phase 1d.
async fn cmd_cluster_ping(
name: &str,
peer: &str,
rpc_addr: SocketAddr,
payload: &str,
) -> Result<()> {
use cluster::transport::{ping, NodeIdentity, QuicClient};
// Pull a fresh ephemeral CA + our own leaf + the peer's leaf. This
// matches the shape a persistent NodeIdentity will have when loaded
// from disk in Phase 1d.
let (id_self, _id_peer) = NodeIdentity::generate_test_pair(name, peer)?;
let client = QuicClient::new("0.0.0.0:0".parse()?, id_self)?;
let conn = client.connect(rpc_addr, peer).await?;
let response = ping(&conn, payload.as_bytes()).await?;
println!("→ sent: {}", payload);
println!("← recv: {}", String::from_utf8_lossy(&response));
conn.close(quinn::VarInt::from_u32(0), b"done");
client.shutdown().await;
Ok(())
}
// ── cluster status ─────────────────────────────────────────────────────────── // ── cluster status ───────────────────────────────────────────────────────────
async fn cmd_cluster_status(cfg: &Config, name: &str, wait_secs: u64) -> Result<()> { async fn cmd_cluster_status(cfg: &Config, name: &str, wait_secs: u64) -> Result<()> {