rtx-csm: Phase 13.5 — emotion-aware LLM prompting
Composes Phase 13.4 reactive emotion with LLM message construction so
the assistant's RESPONSE TEXT adapts to detected user tone, not just
TTS prosody.
--emotion-aware-llm flag (requires --reactive-emotion). When a non-
Neutral label is detected, the LLM-facing copy of the user message is
augmented with `\n\n[user audio tone: {label} — adjust your response
in tone and content to match]`. Per-turn only — the augmentation lives
in history_clone, never in the persistent history, so subsequent turns
aren't biased by stale signals.
Verified end-to-end: server with Q8 + LoRA + reactive-emotion +
emotion-aware-llm booted, bench turn completed 0 errors. Mock LLM
echoed back the augmented text (taking ~44s of TTS), confirming the
annotation reached the LLM. Lib suite 110/110.
The full reactive voice loop now adapts both prosody (TTS emotion_hint)
AND content (LLM annotated user message) to detected user tone. Both
paths flow through the same EmotionDetector trait, so when emotion2vec_
plus_base lands the placeholder swaps cleanly.
Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
@@ -106,6 +106,15 @@ struct Cli {
|
|||||||
#[arg(long, default_value_t = false)]
|
#[arg(long, default_value_t = false)]
|
||||||
reactive_emotion: bool,
|
reactive_emotion: bool,
|
||||||
|
|
||||||
|
/// When `--reactive-emotion` produces a non-Neutral label, ALSO append
|
||||||
|
/// a per-turn signal to the LLM-facing user message so the LLM's
|
||||||
|
/// response text adapts (not just TTS prosody). Per turn only — the
|
||||||
|
/// annotation is NOT pushed to persistent history, so subsequent
|
||||||
|
/// turns aren't biased by stale signals. Has no effect without
|
||||||
|
/// `--reactive-emotion`.
|
||||||
|
#[arg(long, default_value_t = false)]
|
||||||
|
emotion_aware_llm: bool,
|
||||||
|
|
||||||
/// Bearer token required on the WebSocket Authorization header. If
|
/// Bearer token required on the WebSocket Authorization header. If
|
||||||
/// unset the server is open (suitable for local dev only). Reads
|
/// unset the server is open (suitable for local dev only). Reads
|
||||||
/// RTX_AUTH_TOKEN env var if not provided.
|
/// RTX_AUTH_TOKEN env var if not provided.
|
||||||
@@ -468,6 +477,11 @@ struct Shared {
|
|||||||
/// reactive-emotion conditioning. `emotion_hint` above is the fallback
|
/// reactive-emotion conditioning. `emotion_hint` above is the fallback
|
||||||
/// when detection yields Neutral.
|
/// when detection yields Neutral.
|
||||||
reactive_emotion: bool,
|
reactive_emotion: bool,
|
||||||
|
/// When true (and `reactive_emotion` is also true), append a per-turn
|
||||||
|
/// emotion-signal annotation to the LLM-facing user message so the
|
||||||
|
/// response text adapts in addition to the TTS prosody. Per-turn,
|
||||||
|
/// not pushed to persistent history (Phase 13.5).
|
||||||
|
emotion_aware_llm: bool,
|
||||||
metrics: Metrics,
|
metrics: Metrics,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -828,6 +842,7 @@ async fn main() -> Result<()> {
|
|||||||
},
|
},
|
||||||
emotion_hint: cli.emotion_hint.clone(),
|
emotion_hint: cli.emotion_hint.clone(),
|
||||||
reactive_emotion: cli.reactive_emotion,
|
reactive_emotion: cli.reactive_emotion,
|
||||||
|
emotion_aware_llm: cli.emotion_aware_llm,
|
||||||
metrics: Metrics::default(),
|
metrics: Metrics::default(),
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -1299,7 +1314,9 @@ async fn handle_connection(mut socket: WebSocket, shared: Arc<Shared>) {
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
let resolved_hint = reactive_tag.or_else(|| shared.emotion_hint.clone());
|
let resolved_hint = reactive_tag
|
||||||
|
.clone()
|
||||||
|
.or_else(|| shared.emotion_hint.clone());
|
||||||
|
|
||||||
// -- LLM + TTS: stream response back as audio chunks --------------
|
// -- LLM + TTS: stream response back as audio chunks --------------
|
||||||
history.push(ChatMessage::user(&user_text));
|
history.push(ChatMessage::user(&user_text));
|
||||||
@@ -1327,7 +1344,30 @@ async fn handle_connection(mut socket: WebSocket, shared: Arc<Shared>) {
|
|||||||
// forwards to the WebSocket while concurrently watching for
|
// forwards to the WebSocket while concurrently watching for
|
||||||
// user-audio frames (barge-in).
|
// user-audio frames (barge-in).
|
||||||
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<Vec<u8>>();
|
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<Vec<u8>>();
|
||||||
let history_clone = history.clone();
|
|
||||||
|
// Build the LLM-facing history. When --emotion-aware-llm is on AND
|
||||||
|
// we have a non-Neutral reactive tag (Phase 13.4), augment THIS
|
||||||
|
// turn's user message with a tone signal so the LLM adapts its
|
||||||
|
// text. The augmented message is per-turn-only — it never lands
|
||||||
|
// in persistent `history`, so subsequent turns aren't biased by
|
||||||
|
// stale signals. Phase 13.5.
|
||||||
|
let history_clone = if shared.emotion_aware_llm && reactive_tag.is_some() {
|
||||||
|
let tag = reactive_tag.as_deref().unwrap();
|
||||||
|
// Strip surrounding [] from the tag for the prose annotation.
|
||||||
|
let label = tag.trim_matches(|c| c == '[' || c == ']');
|
||||||
|
let mut h = history.clone();
|
||||||
|
if let Some(last) = h.last_mut() {
|
||||||
|
if matches!(last.role, rtx_csm::llm_client::Role::User) {
|
||||||
|
last.content = format!(
|
||||||
|
"{}\n\n[user audio tone: {label} — adjust your response in tone and content to match]",
|
||||||
|
last.content
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
h
|
||||||
|
} else {
|
||||||
|
history.clone()
|
||||||
|
};
|
||||||
let metrics_for_tts = shared.clone();
|
let metrics_for_tts = shared.clone();
|
||||||
// Capture the moment conv_fut starts so we can derive
|
// Capture the moment conv_fut starts so we can derive
|
||||||
// llm_to_first_audio (LLM stream + first-sentence TTS) and
|
// llm_to_first_audio (LLM stream + first-sentence TTS) and
|
||||||
|
|||||||
Reference in New Issue
Block a user