Phase 8b: LAN-first probe with tailnet fallback
Build with clawstor cache / Cargo build (clawstor-cached) (pull_request) Failing after 2s

Second Phase 8 slice. Prior transport.connect() took a single
address; the LAN-first-then-Tailscale routing the arch doc calls
out was implicit ("pick lan_addr OR tailscale_addr from gossip
state") and never actually raced or fell through.

New: QuicClient::connect_lan_first(name, lan, tailscale, lan_probe)
* Try LAN first with `lan_probe` deadline (fleet default ~200ms).
* If LAN handshake fails OR the deadline fires → fall back to
  the Tailscale address.
* Both slots None → error immediately (no hang).

Returns (connection, ConnectRoute) so callers + telemetry see
which side won. New enum ConnectRoute::{Lan(addr), Tailscale(addr)}.

+3 tests exercising the three shapes:
- lan-first when LAN reachable (never dials fake tailscale addr)
- fallback when LAN black-holes (240.0.0.1 SYN gets no response;
  probe deadline fires, tailscale server wins)
- errors cleanly when both addrs absent

375 tests pass (+3). Pre-existing macOS
hot::tests::test_project_target_size_bytes failure unchanged.

Follow-ons for Phase 8 completion:
- Wire the peer-connect call sites (RPC forwarding, PeerStatus,
  build-cache) through connect_lan_first with per-peer
  lan/tailscale addrs from gossip state.
- Document the roaming-client config template.
This commit is contained in:
Omar Sobh
2026-07-14 11:12:19 -07:00
parent 98036f2597
commit ef60a7984e
+157
View File
@@ -524,6 +524,61 @@ impl QuicClient {
.with_context(|| format!("completing handshake with {addr}"))
}
/// Phase 8b (2026-07-14): LAN-first probe with a tailnet
/// fallback. Attempts `lan` (when given) under a tight budget;
/// if the handshake doesn't complete in `lan_probe`, falls
/// back to `tailscale` (when given) under a longer budget.
///
/// Rationale: on the fleet's 1G LAN a handshake typically
/// finishes in single-digit ms. Tailscale (WireGuard over WAN
/// for roaming clients) can take 100-500 ms. A short LAN
/// probe lets in-office nodes take the fast path without
/// starving roaming nodes when LAN isn't reachable.
///
/// If both address slots are `None` the call errors
/// immediately rather than hanging.
pub async fn connect_lan_first(
&self,
expected_server_name: &str,
lan: Option<SocketAddr>,
tailscale: Option<SocketAddr>,
lan_probe: std::time::Duration,
) -> Result<(quinn::Connection, ConnectRoute)> {
if let Some(lan_addr) = lan {
match tokio::time::timeout(
lan_probe,
self.connect(lan_addr, expected_server_name),
)
.await
{
Ok(Ok(conn)) => return Ok((conn, ConnectRoute::Lan(lan_addr))),
Ok(Err(e)) => tracing::debug!(
peer = expected_server_name,
lan = %lan_addr,
error = %e,
"LAN dial failed; trying tailnet if configured"
),
Err(_) => tracing::debug!(
peer = expected_server_name,
lan = %lan_addr,
"LAN probe hit deadline; falling back to tailnet"
),
}
}
if let Some(ts_addr) = tailscale {
let conn = self
.connect(ts_addr, expected_server_name)
.await
.with_context(|| format!("tailnet fallback to {ts_addr}"))?;
return Ok((conn, ConnectRoute::Tailscale(ts_addr)));
}
bail!(
"no reachable address for {expected_server_name}: LAN {} + tailnet {} both failed or absent",
lan.map(|a| a.to_string()).unwrap_or_else(|| "-".into()),
tailscale.map(|a| a.to_string()).unwrap_or_else(|| "-".into())
)
}
/// Graceful shutdown.
pub async fn shutdown(&self) {
self.endpoint
@@ -532,6 +587,16 @@ impl QuicClient {
}
}
/// Phase 8b (2026-07-14): which route won the LAN-first probe.
/// Returned from [`QuicClient::connect_lan_first`] so operators
/// (and telemetry) can see which side of the network was chosen
/// per connection.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConnectRoute {
Lan(SocketAddr),
Tailscale(SocketAddr),
}
/// Open a bidi stream on `conn`, send `payload`, read back the peer's
/// response (bounded by [`MAX_MESSAGE_BYTES`]). This is the client
/// side of the ping RPC.
@@ -695,6 +760,98 @@ mod tests {
accept_task.abort();
}
#[tokio::test]
async fn connect_lan_first_takes_lan_when_reachable() {
// Live LAN server, valid tailscale would just be a decoy —
// we should never dial it. Assert the returned route.
let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
let server = QuicServer::bind(loopback(0), id_b).unwrap();
let server_addr = server.local_addr().unwrap();
let accept_task = tokio::spawn(async move {
if let Some(res) = server.accept().await {
let conn = res.expect("accept");
let _ = ping_handler_loop(conn).await;
}
server.shutdown().await;
});
let client = QuicClient::new(loopback(0), id_a).unwrap();
// Fake tailscale addr = a port nothing binds. Must not be
// dialed since LAN succeeded first.
let fake_ts: SocketAddr = "127.0.0.1:1".parse().unwrap();
let (conn, route) = client
.connect_lan_first(
"b",
Some(server_addr),
Some(fake_ts),
Duration::from_secs(2),
)
.await
.expect("connect_lan_first");
assert!(matches!(route, ConnectRoute::Lan(a) if a == server_addr));
let response = ping(&conn, b"hi").await.unwrap();
assert_eq!(response, b"pong:hi");
conn.close(VarInt::from_u32(0), b"done");
client.shutdown().await;
tokio::time::sleep(Duration::from_millis(50)).await;
accept_task.abort();
}
#[tokio::test]
async fn connect_lan_first_falls_through_to_tailscale_on_lan_deadline() {
// LAN addr is a black hole (drops SYNs). Probe budget = 100ms.
// Tailscale addr = real server. Must fall through and pick
// the tailscale route.
let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
let server = QuicServer::bind(loopback(0), id_b).unwrap();
let server_addr = server.local_addr().unwrap();
let accept_task = tokio::spawn(async move {
if let Some(res) = server.accept().await {
let conn = res.expect("accept");
let _ = ping_handler_loop(conn).await;
}
server.shutdown().await;
});
// 240.x.x.x is RFC1112 unroutable — SYN just times out.
// We don't need a live server; the deadline must fire on
// its own.
let black_hole: SocketAddr = "240.0.0.1:1".parse().unwrap();
let client = QuicClient::new(loopback(0), id_a).unwrap();
let started = std::time::Instant::now();
let (conn, route) = client
.connect_lan_first(
"b",
Some(black_hole),
Some(server_addr),
Duration::from_millis(150),
)
.await
.expect("connect_lan_first");
assert!(matches!(route, ConnectRoute::Tailscale(a) if a == server_addr));
// Sanity: we shouldn't have waited far beyond the probe
// budget before starting the tailscale attempt.
assert!(
started.elapsed() < Duration::from_secs(3),
"took too long: {:?}",
started.elapsed()
);
conn.close(VarInt::from_u32(0), b"done");
client.shutdown().await;
tokio::time::sleep(Duration::from_millis(50)).await;
accept_task.abort();
}
#[tokio::test]
async fn connect_lan_first_errors_when_both_addrs_absent() {
let (id_a, _id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
let client = QuicClient::new(loopback(0), id_a).unwrap();
let err = client
.connect_lan_first("b", None, None, Duration::from_millis(50))
.await
.unwrap_err();
assert!(err.to_string().contains("no reachable address"));
client.shutdown().await;
}
#[tokio::test]
async fn client_rejects_peer_with_wrong_ca() {
// Two CAs, A and X. A pair (id_a, id_b) share CA_A; a rogue id_x