Merge: B4.1 rootfs builder — and the finding that blocks B4.2
Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Omar Sobh
Executable
+171
@@ -0,0 +1,171 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Turn a Docker image into a Firecracker rootfs, and prove a VM boots from it.
|
||||||
|
#
|
||||||
|
# Phase B4. Until now microVMs booted Firecracker's CI Ubuntu image with a
|
||||||
|
# python guest agent bolted on: no git, no toolchain, no CLI. That is fine for
|
||||||
|
# proving vsock works and useless for running a mission.
|
||||||
|
#
|
||||||
|
# Building FROM a Docker image rather than debootstrapping a new one is the
|
||||||
|
# point. The per-CLI images (agent-claude / agent-kimi / agent-glm, per A6) are
|
||||||
|
# already Dockerfiles with a tested env contract — CLI on PATH, credentials in
|
||||||
|
# known places, toolchain present. Rebuilding that as a VM image by hand would
|
||||||
|
# mean maintaining the same facts twice and discovering the drift in production.
|
||||||
|
# `docker export` flattens the image to a tar; this wraps it in an ext4 and adds
|
||||||
|
# the guest agent.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# scripts/fc-build-rootfs.sh <ssh-host> <docker-image> [out-name] [size]
|
||||||
|
# scripts/fc-build-rootfs.sh osobh@tank clawmates/agent-base:dev agent-base 4G
|
||||||
|
#
|
||||||
|
# The result lands at $WORK/rootfs-<out-name>.ext4 on the node. It is NOT made
|
||||||
|
# the default: `vm_create` still boots `rootfs.ext4`, and repointing that is a
|
||||||
|
# deliberate act (B4.3 adds per-mission image selection).
|
||||||
|
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
WORK="${FC_WORK:-/opt/clawmates-fc}"
|
||||||
|
FAILURES=0
|
||||||
|
pass() { printf 'PASS %s\n' "$*"; }
|
||||||
|
fail() { printf 'FAIL %s\n' "$*"; FAILURES=$((FAILURES + 1)); }
|
||||||
|
die() { printf 'ABORT %s\n' "$*" >&2; exit 2; }
|
||||||
|
|
||||||
|
[ $# -ge 2 ] || die "usage: $0 <ssh-host> <docker-image> [out-name] [size]"
|
||||||
|
HOST="$1"
|
||||||
|
IMAGE="$2"
|
||||||
|
NAME="${3:-$(printf '%s' "$2" | tr '/:' '--')}"
|
||||||
|
SIZE="${4:-4G}"
|
||||||
|
OUT="$WORK/rootfs-$NAME.ext4"
|
||||||
|
|
||||||
|
echo "── building $OUT on $HOST from $IMAGE ──"
|
||||||
|
|
||||||
|
# 1. Flatten the image to a tar and unpack it into an ext4.
|
||||||
|
#
|
||||||
|
# `docker export` on a created (never started) container gives the filesystem
|
||||||
|
# with none of the image's metadata — no ENV, no ENTRYPOINT, no WORKDIR. That
|
||||||
|
# metadata matters: a CLI that relies on ENV PATH or ENV HOME would silently
|
||||||
|
# behave differently in the VM. It is extracted separately below and written
|
||||||
|
# into the guest's profile, rather than left to be discovered later.
|
||||||
|
built=$(ssh "$HOST" "
|
||||||
|
set -e
|
||||||
|
cd '$WORK'
|
||||||
|
docker image inspect '$IMAGE' >/dev/null 2>&1 || docker pull -q '$IMAGE' >/dev/null
|
||||||
|
cid=\$(docker create '$IMAGE' /bin/true)
|
||||||
|
trap 'docker rm -f \$cid >/dev/null 2>&1 || true' EXIT
|
||||||
|
rm -f '$OUT' export.tar
|
||||||
|
docker export \"\$cid\" -o export.tar
|
||||||
|
echo \"export=\$(du -m export.tar | cut -f1)MB\"
|
||||||
|
|
||||||
|
# The ext4 is created empty and filled via a mount rather than \`mkfs -d\`:
|
||||||
|
# -d cannot handle device nodes or hard links that a container image may
|
||||||
|
# contain, and fails late and cryptically when it hits one.
|
||||||
|
truncate -s '$SIZE' '$OUT'
|
||||||
|
mkfs.ext4 -q -F '$OUT'
|
||||||
|
sudo mkdir -p /mnt/fcbuild
|
||||||
|
sudo mount -o loop '$OUT' /mnt/fcbuild
|
||||||
|
sudo tar -xf export.tar -C /mnt/fcbuild
|
||||||
|
rm -f export.tar
|
||||||
|
|
||||||
|
# Image metadata the export dropped. Written to /etc/profile.d so both a
|
||||||
|
# login shell and the agent's \`sh -c\` see it.
|
||||||
|
docker image inspect '$IMAGE' --format '{{range .Config.Env}}export {{.}}
|
||||||
|
{{end}}' | sudo tee /mnt/fcbuild/etc/profile.d/00-image-env.sh >/dev/null
|
||||||
|
sudo chmod 0644 /mnt/fcbuild/etc/profile.d/00-image-env.sh
|
||||||
|
echo \"env_vars=\$(wc -l < /mnt/fcbuild/etc/profile.d/00-image-env.sh)\"
|
||||||
|
" 2>&1) || { fail "could not build the filesystem: $(printf '%s' "$built" | tail -3)"; ssh "$HOST" "sudo umount /mnt/fcbuild 2>/dev/null; true"; exit 1; }
|
||||||
|
pass "unpacked $IMAGE into $OUT ($(printf '%s' "$built" | tr '\n' ' '))"
|
||||||
|
|
||||||
|
# 2. Install the guest agent and init.
|
||||||
|
#
|
||||||
|
# Not a systemd unit: a `docker export` rootfs usually has no init system at
|
||||||
|
# all, and adding one to boot a single agent would be a large amount of surface
|
||||||
|
# for no benefit. `init=` runs the agent as pid 1 directly — and pid 1 must
|
||||||
|
# never exit, so the init script execs it rather than backgrounding it.
|
||||||
|
agent=$(ssh "$HOST" "
|
||||||
|
set -e
|
||||||
|
# python3 is how the agent is written; a rootfs without it cannot serve.
|
||||||
|
test -x /mnt/fcbuild/usr/bin/python3 || test -x /mnt/fcbuild/usr/local/bin/python3 \
|
||||||
|
|| { echo 'NO-PYTHON3'; exit 1; }
|
||||||
|
sudo cp /mnt/fcbuild/usr/local/bin/fcagent /tmp/.probe 2>/dev/null || true
|
||||||
|
# Reuse the agent already baked into the golden rootfs so there is ONE copy of
|
||||||
|
# this protocol on the node, not two that can drift.
|
||||||
|
sudo mkdir -p /mnt/fcgolden && sudo mount -o loop,ro '$WORK/rootfs.ext4' /mnt/fcgolden
|
||||||
|
sudo cp /mnt/fcgolden/usr/local/bin/fcagent /mnt/fcbuild/usr/local/bin/fcagent
|
||||||
|
sudo cp /mnt/fcgolden/usr/local/bin/fcinit /mnt/fcbuild/usr/local/bin/fcinit
|
||||||
|
sudo umount /mnt/fcgolden && sudo rmdir /mnt/fcgolden
|
||||||
|
sudo chmod 0755 /mnt/fcbuild/usr/local/bin/fcagent /mnt/fcbuild/usr/local/bin/fcinit
|
||||||
|
sudo umount /mnt/fcbuild
|
||||||
|
echo installed
|
||||||
|
" 2>&1)
|
||||||
|
case "$agent" in
|
||||||
|
*installed*) pass "guest agent installed (shared with the golden rootfs)" ;;
|
||||||
|
*NO-PYTHON3*) fail "$IMAGE has no python3 — the guest agent cannot run in it"; ssh "$HOST" "sudo umount /mnt/fcbuild 2>/dev/null; true" ;;
|
||||||
|
*) fail "could not install the guest agent: $(printf '%s' "$agent" | tail -2)"; ssh "$HOST" "sudo umount /mnt/fcbuild 2>/dev/null; true" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# 3. Boot it. An image that builds and cannot boot is worse than no image,
|
||||||
|
# because it looks finished. Everything below runs as the daemon user.
|
||||||
|
if [ "$FAILURES" -eq 0 ]; then
|
||||||
|
boot=$(ssh "$HOST" "
|
||||||
|
cd '$WORK'
|
||||||
|
cp --sparse=always '$OUT' probe-rootfs.ext4
|
||||||
|
cat > probe-vm.json <<JSON
|
||||||
|
{
|
||||||
|
\"boot-source\": {
|
||||||
|
\"kernel_image_path\": \"$WORK/vmlinux\",
|
||||||
|
\"boot_args\": \"console=ttyS0 reboot=k panic=1 pci=off init=/usr/local/bin/fcinit\"
|
||||||
|
},
|
||||||
|
\"drives\": [{\"drive_id\":\"rootfs\",\"path_on_host\":\"$WORK/probe-rootfs.ext4\",\"is_root_device\":true,\"is_read_only\":false}],
|
||||||
|
\"machine-config\": {\"vcpu_count\":2,\"mem_size_mib\":2048,\"smt\":false},
|
||||||
|
\"vsock\": {\"guest_cid\": 3, \"uds_path\": \"$WORK/probe.sock\"}
|
||||||
|
}
|
||||||
|
JSON
|
||||||
|
rm -f probe.sock probe.log
|
||||||
|
setsid timeout 40 firecracker --no-api --config-file probe-vm.json > probe.log 2>&1 &
|
||||||
|
PG=\$!
|
||||||
|
for i in \$(seq 1 400); do grep -qa FC-AGENT-LISTENING probe.log && break; sleep 0.05; done
|
||||||
|
python3 - <<'PY' 2>&1 | tail -6
|
||||||
|
import socket, struct, json
|
||||||
|
def call(cmd):
|
||||||
|
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||||
|
s.connect('$WORK/probe.sock')
|
||||||
|
s.sendall(b'CONNECT 9001\n'); s.recv(64)
|
||||||
|
req = json.dumps({'op':'exec','cmd':cmd,'timeout':60}).encode()
|
||||||
|
s.sendall(struct.pack('>I', len(req)) + req)
|
||||||
|
n = struct.unpack('>I', s.recv(4))[0]
|
||||||
|
buf = b''
|
||||||
|
while len(buf) < n: buf += s.recv(n - len(buf))
|
||||||
|
s.close()
|
||||||
|
return json.loads(buf)
|
||||||
|
# What a mission actually needs, asked of the image rather than assumed.
|
||||||
|
for label, cmd in [('git', 'git --version'),
|
||||||
|
('shell-env', '. /etc/profile.d/00-image-env.sh 2>/dev/null; echo PATH=\$PATH'),
|
||||||
|
('write', 'mkdir -p /mission && echo ok > /mission/x && cat /mission/x')]:
|
||||||
|
try:
|
||||||
|
r = call(cmd)
|
||||||
|
print('%s rc=%s %s' % (label, r.get('rc'), (r.get('stdout') or r.get('stderr') or '').strip()[:90]))
|
||||||
|
except Exception as e:
|
||||||
|
print('%s UNREACHABLE %s' % (label, e))
|
||||||
|
PY
|
||||||
|
kill -- -\$PG 2>/dev/null
|
||||||
|
rm -f probe.sock probe-rootfs.ext4 probe-vm.json
|
||||||
|
" 2>&1)
|
||||||
|
|
||||||
|
printf '%s\n' "$boot" | sed 's/^/ /'
|
||||||
|
case "$boot" in
|
||||||
|
*"git rc=0"*) pass "the built rootfs boots and has git" ;;
|
||||||
|
*UNREACHABLE*) fail "the built rootfs booted but its agent was unreachable" ;;
|
||||||
|
*) fail "the built rootfs did not provide git" ;;
|
||||||
|
esac
|
||||||
|
case "$boot" in
|
||||||
|
*"write rc=0"*) pass "the guest can write to /mission" ;;
|
||||||
|
*) fail "the guest could not write to /mission" ;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
if [ "$FAILURES" -gt 0 ]; then
|
||||||
|
echo "$FAILURES check(s) failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
ssh "$HOST" "ls -lh '$OUT' | awk '{print \$5\" \"\$9}'"
|
||||||
|
echo "rootfs ready (not yet the default — vm_create still boots rootfs.ext4)"
|
||||||
Reference in New Issue
Block a user