#!/usr/bin/env bash # Roll N seeds through `emotional_speech.sh`, score each with # `examples/quality_eval`, copy the lowest-WER output to --out. # # Why: Sprint 2 reproducibility bench (2026-04-29) showed the recipe # has high seed variance for text content fidelity. Same prompt # rendered "Today, today I want to share..." (WER 0.71) at seed 7 # and "The police are, if you're..." (WER 2.0) at seed 42. Speaker # cosine is consistently elevated across seeds (the steering shifts # voice character reliably), but content is roll-the-dice. This # wrapper rolls the dice multiple times automatically and picks the # winner. # # Cost: N × the single-shot cost. Defaults to 5 seeds. Lower N if # the recipe is reliable for your prompt category; raise if you need # the cleanest possible output. # # Usage: # scripts/emotional_speech_n.sh \ # --text "..." \ # --context-wav .wav --context-text "..." \ # --emotion happy --steering-dir /tmp/ravdess_steering \ # --out /tmp/best.wav \ # [--seeds 42,7,100,123,256] # [--wavlm-sv /tmp/wavlm_sv.safetensors] # [--keep-all] # keep all candidates in /candidates/ # [other flags pass through to emotional_speech.sh] set -euo pipefail TEXT="" CTX_WAV="" CTX_TEXT="" EMOTION="" STEERING_DIR="" OUT="" SEEDS="42,7,100,123,256" WAVLM_SV="/tmp/wavlm_sv.safetensors" KEEP_ALL=false PASSTHROUGH=() while [[ $# -gt 0 ]]; do case "$1" in --text) TEXT="$2"; shift 2 ;; --context-wav) CTX_WAV="$2"; shift 2 ;; --context-text) CTX_TEXT="$2"; shift 2 ;; --emotion) EMOTION="$2"; shift 2 ;; --steering-dir) STEERING_DIR="$2"; shift 2 ;; --out) OUT="$2"; shift 2 ;; --seeds) SEEDS="$2"; shift 2 ;; --wavlm-sv) WAVLM_SV="$2"; shift 2 ;; --keep-all) KEEP_ALL=true; shift 1 ;; -h|--help) sed -n '2,30p' "$0"; exit 0 ;; # Pass any unknown flags through to emotional_speech.sh *) PASSTHROUGH+=("$1"); shift 1 ;; esac done for v in TEXT CTX_WAV CTX_TEXT EMOTION STEERING_DIR OUT; do if [[ -z "${!v}" ]]; then echo "error: --${v,,} unset" 2>/dev/null || true echo "usage: $0 --text T --context-wav W --context-text C --emotion E --steering-dir D --out O [--seeds 42,7,100]" >&2 exit 1 fi done if [[ ! -f "$WAVLM_SV" ]]; then echo "wavlm-sv not found: $WAVLM_SV" >&2; exit 1 fi SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" WORKSPACE_DIR="$(cd "$SCRIPT_DIR/../../../.." && pwd)" EVAL_BIN="$WORKSPACE_DIR/target/release/examples/quality_eval" EMO_SCRIPT="$SCRIPT_DIR/emotional_speech.sh" for f in "$EVAL_BIN" "$EMO_SCRIPT"; do if [[ ! -x "$f" ]]; then echo "missing: $f" >&2 exit 1 fi done # Working directory derived from --out path. OUT_DIR="$(dirname "$OUT")" mkdir -p "$OUT_DIR" CAND_DIR="$OUT_DIR/candidates_$(date +%s%N)" mkdir -p "$CAND_DIR" trap '[[ "$KEEP_ALL" == false ]] && rm -rf "$CAND_DIR"' EXIT # Comma-split seeds into an array. IFS=',' read -ra SEED_ARR <<< "$SEEDS" echo "→ rolling ${#SEED_ARR[@]} seeds for emotion='$EMOTION' on:" echo " \"$(echo "$TEXT" | head -c 70)...\"" # Generate one candidate per seed. PAIRS_FILE="$CAND_DIR/eval_pairs.jsonl" : > "$PAIRS_FILE" for seed in "${SEED_ARR[@]}"; do candidate="$CAND_DIR/seed_${seed}.wav" "$EMO_SCRIPT" \ --text "$TEXT" \ --context-wav "$CTX_WAV" --context-text "$CTX_TEXT" \ --emotion "$EMOTION" --steering-dir "$STEERING_DIR" \ --seed "$seed" \ --out "$candidate" \ ${PASSTHROUGH[@]+"${PASSTHROUGH[@]}"} 2>&1 | grep -E "recipe|generated" | sed "s/^/ [seed=$seed] /" || true # Skip rows for outputs that didn't exist (e.g. rare crashes). if [[ -f "$candidate" ]]; then jq -nc --arg ref "$CTX_WAV" --arg gen "$candidate" \ --arg t "$TEXT" --arg s "$seed" \ '{ref_wav: $ref, gen_wav: $gen, ref_text: $t, seed: $s}' >> "$PAIRS_FILE" fi done # Score everything in one quality_eval pass (reuses model load). SCORES_FILE="$CAND_DIR/scores.jsonl" "$EVAL_BIN" \ --in "$PAIRS_FILE" \ --out "$SCORES_FILE" \ --wavlm-sv "$WAVLM_SV" 2>&1 | tail -3 echo echo "=== candidates (lowest WER wins, ties → max cosine) ===" # Null-safe formatting: clip-too-short cases set cosine = -1; treat # missing/null metrics as -1 / 99 in the table so the row still prints. jq -r '[ .seed, ((.metrics.speaker_cosine // -1) * 1000 | round / 1000), ((.metrics.wer // 99) * 1000 | round / 1000), ((.metrics.transcript // "(no transcript)")[0:60]) ] | @tsv' "$SCORES_FILE" \ | sort -t $'\t' -k3,3g -k2,2gr \ | awk -F'\t' 'BEGIN{printf "%-6s %-8s %-8s %s\n","seed","cos","WER","transcript"} {printf "%-6s %-8s %-8s %s\n",$1,$2,$3,$4}' # Pick winner via jq -s, treating missing WER as +infinity so failed # generations sink to the bottom. WINNER_PATH=$(jq -s 'map(. + {_wer: (.metrics.wer // 99), _cos: (.metrics.speaker_cosine // -1)}) | sort_by(._wer, -._cos) | .[0].gen_wav' "$SCORES_FILE" | tr -d '"') WINNER_SEED=$(jq -s 'map(. + {_wer: (.metrics.wer // 99), _cos: (.metrics.speaker_cosine // -1)}) | sort_by(._wer, -._cos) | .[0].seed' "$SCORES_FILE" | tr -d '"') if [[ -z "$WINNER_PATH" || ! -f "$WINNER_PATH" ]]; then echo "error: no candidate to copy as winner" >&2 exit 2 fi cp "$WINNER_PATH" "$OUT" echo echo "✓ winner seed=$WINNER_SEED → $OUT" if [[ "$KEEP_ALL" == true ]]; then echo " all candidates kept in $CAND_DIR/" fi