#!/usr/bin/env bash # One-shot emotional voice synthesis using the Phase 9 recipe: # - Selective CFG schedule `linear:3.0:1.0:25` (robust default) # - Activation steering @ scale 0.3 on mid layers [8, 10, 12] # - Reference-clip context conditioning # # This is the codified version of the empirical findings logged in # `docs/perf_history.md` Phase 9. Recipe was tuned for cross-character # emotional speech (e.g. take a neutral lecture voice, render it angry). # # Steering vectors must be pre-extracted via # `examples/steering_extract` from a labeled emotion corpus (RAVDESS, # CREMA-D, ESD, etc). Use `scripts/build_ravdess_manifest.sh` to # convert RAVDESS into a manifest first. # # Usage: # scripts/emotional_speech.sh \ # --text "What you want said." \ # --context-wav .wav \ # --context-text "Transcript of the reference clip." \ # --emotion angry \ # --steering-dir /tmp/ravdess_steering \ # --out /tmp/out.wav \ # [--scale 0.3] # steering scale, lower = more text fidelity # [--cfg-schedule linear:3.0:1.0:25] # [--layers 8,10,12] # # Recipe defaults are deliberate; override only when you have a reason. # Sprint 2 measurements: # - speaker_cosine vs angry-actor reference rises 0.55 → 0.846 with # scale=0.3 stacked on Selective CFG # - scale ≥ 0.5 corrupts the output (Moonshine WER 1.0 on Unicode noise) # - layers including 0–7 also corrupt (input embedding fidelity loss) # # Per-emotion scale sweet spots (RAVDESS-extracted, mid-layer recipe, # linear:3.0:1.0:25 CFG, single-prompt single-seed measurement — treat # as starting points, not guarantees): # - happy: 0.30 → fluent speech # - angry: 0.20 → fluent speech (0.30 produces music-token outputs) # - fearful: 0.20 → fluent speech (0.30 produces music-token outputs) # - sad: no robust scale found — model resists "sad" steering on # most prompts; one-word outputs at all tested scales # We leave --scale defaulted to 0.3 because the wrapper can't know which # emotion is being requested at default-construction time. Override # with --scale 0.2 when using angry or fearful. set -euo pipefail TEXT="" CTX_WAV="" CTX_TEXT="" EMOTION="" STEERING_DIR="" OUT="" TARGET="decoder" # Default to decoder route (Phase 9 winner: preserves text fidelity). SCALE="" # if unset, picked per-(target,emotion) below CFG_SCHEDULE="linear:3.0:1.0:25" LAYERS="" # if unset, picked per-target below SPEAKER=0 SEED=42 while [[ $# -gt 0 ]]; do case "$1" in --text) TEXT="$2"; shift 2 ;; --context-wav) CTX_WAV="$2"; shift 2 ;; --context-text) CTX_TEXT="$2"; shift 2 ;; --emotion) EMOTION="$2"; shift 2 ;; --steering-dir) STEERING_DIR="$2"; shift 2 ;; --out) OUT="$2"; shift 2 ;; --target) TARGET="$2"; shift 2 ;; --scale) SCALE="$2"; shift 2 ;; --cfg-schedule) CFG_SCHEDULE="$2"; shift 2 ;; --layers) LAYERS="$2"; shift 2 ;; --speaker) SPEAKER="$2"; shift 2 ;; --seed) SEED="$2"; shift 2 ;; -h|--help) sed -n '2,32p' "$0"; exit 0 ;; *) echo "unknown arg: $1" >&2; exit 1 ;; esac done for v in TEXT CTX_WAV CTX_TEXT EMOTION STEERING_DIR OUT; do if [[ -z "${!v}" ]]; then echo "error: --${v,,} (or one of the required flags) is unset" >&2 echo "usage: $0 --text T --context-wav W --context-text C --emotion E --steering-dir D --out O [--target backbone|decoder]" >&2 exit 1 fi done if [[ "$TARGET" != "backbone" && "$TARGET" != "decoder" ]]; then echo "error: --target must be 'backbone' or 'decoder' (got '$TARGET')" >&2 exit 1 fi STEERING_FILE="$STEERING_DIR/${EMOTION}.safetensors" if [[ ! -f "$STEERING_FILE" ]]; then echo "error: no steering file for emotion '$EMOTION' at $STEERING_FILE" >&2 echo "available emotions in $STEERING_DIR:" >&2 ls "$STEERING_DIR" 2>/dev/null | sed 's/\.safetensors$//' | sed 's/^/ /' >&2 || true exit 1 fi # Per-(target, emotion) scale defaults (Sprint 2 + decoder findings). if [[ -z "$SCALE" ]]; then if [[ "$TARGET" == "decoder" ]]; then # Decoder vectors are ~10× smaller magnitude, so scales are 10× higher. case "$EMOTION" in *) SCALE=1.0 ;; # only happy benched; conservative default for others esac else case "$EMOTION" in angry|fearful|sad) SCALE=0.2 ;; happy) SCALE=0.3 ;; *) SCALE=0.3 ;; esac fi fi # Per-target default layers. if [[ -z "$LAYERS" ]]; then if [[ "$TARGET" == "decoder" ]]; then LAYERS="2,3" # decoder last-two — empirical no-repetition recipe else LAYERS="8,10,12" # backbone mid-deep — empirical fluency-preserving recipe fi fi SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" WORKSPACE_DIR="$(cd "$SCRIPT_DIR/../../../.." && pwd)" GEN_BIN="$WORKSPACE_DIR/target/release/examples/generate" if [[ ! -x "$GEN_BIN" ]]; then echo "error: missing $GEN_BIN" >&2 echo "build with: cargo build -p rtx-csm --release --example generate" >&2 exit 1 fi echo "→ recipe: target=$TARGET emotion=$EMOTION scale=$SCALE cfg=$CFG_SCHEDULE layers=$LAYERS" echo " context: $(basename "$CTX_WAV")" if [[ "$TARGET" == "decoder" ]]; then "$GEN_BIN" \ --text "$TEXT" --speaker "$SPEAKER" --seed "$SEED" \ --context-wav "$CTX_WAV" --context-text "$CTX_TEXT" --context-speaker 1 \ --enable-cfg --cfg-schedule "$CFG_SCHEDULE" \ --decoder-steering-vec "$STEERING_FILE" \ --decoder-steering-scale "$SCALE" \ --decoder-steering-layers "$LAYERS" \ --out "$OUT" else "$GEN_BIN" \ --text "$TEXT" --speaker "$SPEAKER" --seed "$SEED" \ --context-wav "$CTX_WAV" --context-text "$CTX_TEXT" --context-speaker 1 \ --enable-cfg --cfg-schedule "$CFG_SCHEDULE" \ --steering-vec "$STEERING_FILE" \ --steering-scale "$SCALE" \ --steering-layers "$LAYERS" \ --out "$OUT" fi echo "✓ wrote $OUT"