▋
src/bot/replyPipeline.ts

import { clamp, sanitizeBotText } from "../utils.ts"; import { buildReplyPrompt, buildSystemPrompt } from "../prompts/index.ts"; import { getMediaPromptCraftGuidance } from "../prompts/promptCore.ts"; import type { ReplyAttemptOptions } from "../bot.ts"; import { REPLY_OUTPUT_JSON_SCHEMA, composeReplyImagePrompt, composeReplyVideoPrompt, embedWebSearchSources, emptyMentionResolution, parseStructuredReplyOutput, pickReplyMediaDirective, resolveMaxMediaPromptLen, normalizeSkipSentinel, splitDiscordMessage, extractUrlsFromText } from "./botHelpers.ts"; import { getLocalTimeZoneLabel } from "./automation.ts"; import { buildReplyToolSet, executeReplyTool } from "../tools/replyTools.ts"; import type { ReplyToolContext, ReplyToolRuntime, ReplyToolDefinition } from "../tools/replyTools.ts"; import { runModelRequestedWebSearch } from "./replyFollowup.ts"; import { buildTextReplyScopeKey } from "../tools/activeReplyRegistry.ts"; import { isAbortError, throwIfAborted } from "../tools/abortError.ts"; import { buildRuntimeDecisionCorrelation } from "../services/runtimeCorrelation.ts"; import { resolveDeterministicMentions } from "./mentions.ts"; import { MAX_MODEL_IMAGE_INPUTS, UNICODE_REACTIONS, appendReplyFollowupPrompt, buildLoggedReplyPrompts, createReplyPerformanceTracker, createReplyPromptCapture, finalizeReplyPerformanceSample } from "./replyPipelineShared.ts"; import { resolveTextAttentionState } from "./replyAdmission.ts"; import { loadConversationContinuityContext } from "./conversationContinuity.ts"; import { loadBehavioralMemoryFacts } from "./memorySlice.ts"; import { getActivitySettings, getAutomationsSettings, getBotName, getDiscoverySettings, getMemorySettings, getReplyPermissions, getVideoContextSettings, getVisionSettings, getVoiceSettings, isDevTaskEnabled, isDevTaskUserAllowed, isMinecraftEnabled } from "../settings/agentStack.ts"; import type { Settings } from "../settings/settingsSchema.ts"; import { buildContextContentBlocks, type ContentBlock, type ContextMessage } from "../llm/serviceShared.ts"; import { SWARM_TOOL_SCHEMAS, VOICE_TOOL_SCHEMAS } from "../tools/sharedToolSchemas.ts"; import type { ReplyPipelineRuntime } from "./botContext.ts"; import { buildMinecraftSessionScopeKey, findReusableMinecraftSession, getMinecraftSessionPromptHint } from "../agents/minecraft/minecraftSessionAccess.ts"; import { isLikelyImageUrl, parseHistoryImageReference } from "./messageHistory.ts"; import { isLikelyDirectVideoUrl, parseVideoTarget, type VideoTarget } from "../video/videoTargets.ts";

type ReplyPipelineAttachment = { url?: string; proxyURL?: string; name?: string; contentType?: string; };

type ReplyPipelineAttachmentCollection = { size?: number; values: () => IterableIterator; };

type ReplyPipelineEmbed = { url?: string; type?: string; video?: { url?: string; proxyURL?: string; } | null; };

type ReplyPipelineGuildMember = { id?: string; displayName?: string; nickname?: string | null; user?: { id?: string; username?: string; globalName?: string | null; } | null; };

type ReplyPipelineGuild = { members?: { cache?: { size?: number; values: () => IterableIterator; }; search?: (options: { query: string; limit: number; }) => Promise<{ values: () => IterableIterator; }>; } | null; };

type ReplyPipelineMentions = { users?: { size: number; has: (id: string | undefined) => boolean; } | null; repliedUser?: { id: string; } | null; };

type ReplyMessagePayloadFile = { attachment: Buffer; name: string; };

type ReplyMessagePayload = Record<string, unknown> & { content: string; files?: ReplyMessagePayloadFile[]; allowedMentions?: { repliedUser?: boolean; }; };

type ReplyPipelineSentMessage = { id: string; createdTimestamp: number; guildId: string | null; channelId: string; content?: string; attachments?: ReplyPipelineAttachmentCollection; embeds?: ReplyPipelineEmbed[]; };

type ReplyPipelineChannel = { sendTyping: () => Promise; send: (payload: ReplyMessagePayload) => Promise; };

type ReplyPipelineMessage = ReplyPipelineSentMessage & { content: string; guild: ReplyPipelineGuild | null; author: { id: string; username?: string; bot?: boolean; }; member?: { displayName?: string | null; } | null; channel: ReplyPipelineChannel; mentions?: ReplyPipelineMentions; reference?: { messageId?: string; } | null; referencedMessage?: { id?: string; } | null; react: (emoji: string) => Promise; reply: (payload: ReplyMessagePayload) => Promise; };

type ReplyAddressSignal = | ReplyAttemptOptions["addressSignal"] | Awaited<ReturnType<ReplyPipelineRuntime["getReplyAddressSignal"]>>; type ReplyRecentMessage = Record<string, unknown>; type ReplyImageInput = Record<string, unknown> & { url?: string; mediaType?: string; contentType?: string; dataBase64?: string; }; type ReplyVideoInput = Record<string, unknown> & { url?: string; filename?: string; contentType?: string; videoRef?: string; }; type ReplyGeneration = { provider?: string; model?: string; usage?: Record<string, unknown> | null; costUsd?: number | null; text: string; rawContent?: unknown; toolCalls?: Array<{ id: string; name: string; input: Record<string, unknown>; }>; }; type ReplyToolExecutionResult = Awaited<ReturnType>; type ReplyPerformance = ReturnType; type ReplyPromptCaptureState = ReturnType; type ReplyPrompts = ReturnType; type ReplyContinuityContext = Awaited<ReturnType>; type ReplyPromptBase = Parameters[0]; type ReplyTrace = { guildId: string | null; channelId: string; userId: string; source: string | null; event: string | null; reason: string | null; messageId: string | null; }; type ReplyDirective = ReturnType; type ReplyMediaDirective = ReturnType; type ReplyMentionResolution = Awaited<ReturnType>; type ReplyReactionResult = Awaited<ReturnType<ReplyPipelineRuntime["maybeApplyReplyReaction"]>>; type ReplyScreenShareOffer = Awaited<ReturnType<ReplyPipelineRuntime["maybeHandleScreenWatchIntent"]>>; type ReplyWebSearchState = ReturnType<ReplyPipelineRuntime["buildWebSearchContext"]> & { summaryText?: string | null; }; type TurnScopedUrlMediaInputs = { imageInputs: ReplyImageInput[]; videoInputs: ReplyVideoInput[]; }; type TurnVisualMediaInspection = { imageInputs: ReplyImageInput[]; promptText: string; };

const MAX_TURN_SCOPED_URL_IMAGES = MAX_MODEL_IMAGE_INPUTS; const MAX_TURN_SCOPED_URL_VIDEOS = 3; const VISUAL_MEDIA_REQUEST_RE = /\b(?:what|what's|whats|happen(?:s|ing)?|main\s+point|point\s+of|describe|break\s+down|look(?:ing)?|watch(?:ing)?|see(?:ing)?|view(?:ed|ing)?|understand(?:s|ing)?|process(?:ed|ing)?)\b/i; const VISUAL_MEDIA_SUBJECT_RE = /\b(?:gif|video|clip|animation|motion|frames?|this|that|it)\b/i;

function isVisualMediaInspectionRequest(text: string) { const normalized = String(text || "").replace(/\s+/g, " ").trim(); if (!normalized) return false; return VISUAL_MEDIA_REQUEST_RE.test(normalized) && VISUAL_MEDIA_SUBJECT_RE.test(normalized); }

function getRecentMessageId(row: Record<string, unknown> | null | undefined) { return String(row?.message_id || row?.messageId || row?.id || "").trim(); }

function getTurnScopedTextEntries({ message, recentMessages, triggerMessageIds }: { message: ReplyPipelineMessage; recentMessages: ReplyRecentMessage[]; triggerMessageIds: string[]; }) { const triggerSet = new Set( [...triggerMessageIds, String(message.id || "")] .map((value) => String(value || "").trim()) .filter(Boolean) ); const rowsById = new Map<string, ReplyRecentMessage>(); for (const row of Array.isArray(recentMessages) ? recentMessages : []) { const id = getRecentMessageId(row); if (!id || !triggerSet.has(id)) continue; rowsById.set(id, row); }

const entries: Array<{ id: string | null; content: string }> = []; const seen = new Set(); const pushEntry = (id: string | null, content: string) => { const normalizedContent = String(content || "").trim(); if (!normalizedContent) return; const key = id ? id:${id} : content:${normalizedContent}; if (seen.has(key)) return; seen.add(key); entries.push({ id, content: normalizedContent }); };

for (const id of triggerMessageIds) { const row = rowsById.get(String(id || "").trim()); if (row) pushEntry(getRecentMessageId(row) || null, String(row.content || "")); } pushEntry(String(message.id || "").trim() || null, String(message.content || ""));

return entries; }

function deriveFilenameFromUrl(url: string, fallback = "(linked media)") { const normalized = String(url || "").trim(); if (!normalized) return fallback; try { const parsed = new URL(normalized); const segment = parsed.pathname.split("/").filter(Boolean).at(-1) || ""; return decodeURIComponent(segment).trim() || fallback; } catch { const segment = normalized.split("?")[0].split("/").filter(Boolean).at(-1) || ""; return decodeURIComponent(segment).trim() || fallback; } }

function inferMotionContentTypeFromUrl(url: string) { const filename = deriveFilenameFromUrl(url, "").toLowerCase(); if (filename.endsWith(".gif")) return "image/gif"; if (filename.endsWith(".mp4") || filename.endsWith(".m4v")) return "video/mp4"; if (filename.endsWith(".mov")) return "video/quicktime"; if (filename.endsWith(".webm")) return "video/webm"; if (filename.endsWith(".mkv")) return "video/x-matroska"; if (filename.endsWith(".avi")) return "video/x-msvideo"; if (filename.endsWith(".mpeg") || filename.endsWith(".mpg")) return "video/mpeg"; return ""; }

function isLikelyAnimatedImageUrl(url: string) { const text = String(url || "").trim(); if (!text) return false; try { const parsed = new URL(text); const pathname = String(parsed.pathname || "").toLowerCase(); if (pathname.endsWith(".gif")) return true; return String(parsed.searchParams.get("format") || "").trim().toLowerCase() === "gif"; } catch { return false; } }

function safeHostname(url: string) { try { return new URL(String(url || "")).hostname.toLowerCase(); } catch { return ""; } }

function isLikelyMotionMediaUrl(url: string) { return isLikelyDirectVideoUrl(url) || isLikelyAnimatedImageUrl(url); }

function normalizeVideoInput(input: ReplyVideoInput): ReplyVideoInput | null { const url = String(input?.url || "").trim(); if (!url) return null; return { url, filename: String(input?.filename || deriveFilenameFromUrl(url)).trim() || "(linked media)", contentType: String(input?.contentType || inferMotionContentTypeFromUrl(url)).trim() }; }

function mergeVideoInputs(inputs: ReplyVideoInput[], maxInputs = MAX_TURN_SCOPED_URL_VIDEOS) { const out: ReplyVideoInput[] = []; const seen = new Set(); for (const input of inputs) { if (out.length >= maxInputs) break; const normalized = normalizeVideoInput(input); if (!normalized?.url || seen.has(normalized.url)) continue; seen.add(normalized.url); out.push(normalized); } return out; }

function collectTurnScopedUrlMediaInputs({ message, recentMessages, triggerMessageIds, existingImageUrls, existingVideoUrls }: { message: ReplyPipelineMessage; recentMessages: ReplyRecentMessage[]; triggerMessageIds: string[]; existingImageUrls?: string[]; existingVideoUrls?: string[]; }): TurnScopedUrlMediaInputs { const imageInputs: ReplyImageInput[] = []; const videoInputs: ReplyVideoInput[] = []; const seenImages = new Set((existingImageUrls || []).map((url) => String(url || "").trim()).filter(Boolean)); const seenVideos = new Set((existingVideoUrls || []).map((url) => String(url || "").trim()).filter(Boolean)); const addImageUrl = (url: string) => { const normalizedUrl = String(url || "").trim(); if (!normalizedUrl || seenImages.has(normalizedUrl) || imageInputs.length >= MAX_TURN_SCOPED_URL_IMAGES) return; if (isLikelyAnimatedImageUrl(normalizedUrl) || !isLikelyImageUrl(normalizedUrl)) return; const parsed = parseHistoryImageReference(normalizedUrl); seenImages.add(normalizedUrl); imageInputs.push({ url: normalizedUrl, filename: parsed.filename, contentType: parsed.contentType, mediaType: parsed.contentType }); }; const addVideoUrl = (url: string) => { const normalizedUrl = String(url || "").trim(); if (!normalizedUrl || seenVideos.has(normalizedUrl) || videoInputs.length >= MAX_TURN_SCOPED_URL_VIDEOS) return; seenVideos.add(normalizedUrl); videoInputs.push({ url: normalizedUrl, filename: deriveFilenameFromUrl(normalizedUrl), contentType: inferMotionContentTypeFromUrl(normalizedUrl) }); };

for (const entry of getTurnScopedTextEntries({ message, recentMessages, triggerMessageIds })) { const urls = extractUrlsFromText(entry.content) .map((url) => String(url || "").trim()) .filter(Boolean); if (!urls.length) continue;

const hasDirectMotion = urls.some((url) => isLikelyMotionMediaUrl(url));
for (const url of urls) {
  if (isLikelyMotionMediaUrl(url)) {
    addVideoUrl(url);
  } else {
    addImageUrl(url);
  }
}
if (hasDirectMotion) continue;

for (const url of urls) {
  if (videoInputs.length >= MAX_TURN_SCOPED_URL_VIDEOS) break;
  if (parseVideoTarget(url)) addVideoUrl(url);
}

}

return { imageInputs, videoInputs }; }

function buildFallbackVideoTarget(url: string): VideoTarget { return { key: generic:${url}, url, kind: "generic", videoId: null }; }

function normalizeVideoDependencyName(value: unknown): string { const name = String(value || "").trim(); return name === "ffmpeg" || name === "yt-dlp" ? name : ""; }

function getVideoMissingDependencies(video: Record<string, unknown> | null | undefined): string[] { if (!video) return []; const dependencies = new Set(); const rawDependencies = Array.isArray(video.missingDependencies) ? video.missingDependencies : []; for (const dependency of rawDependencies) { const normalized = normalizeVideoDependencyName(dependency); if (normalized) dependencies.add(normalized); }

const codes = [video.keyframeErrorCode, video.transcriptErrorCode].map((value) => String(value || "")); if (codes.includes("missing_ffmpeg")) dependencies.add("ffmpeg"); if (codes.includes("missing_yt_dlp")) dependencies.add("yt-dlp");

const errorText = [video.keyframeError, video.transcriptError] .map((value) => String(value || "").toLowerCase()) .join(" "); if (errorText.includes("local runtime dependency missing") || errorText.includes("not installed")) { if (errorText.includes("ffmpeg")) dependencies.add("ffmpeg"); if (errorText.includes("yt-dlp")) dependencies.add("yt-dlp"); }

return Array.from(dependencies); }

function buildVideoDependencyFailureLine(dependencies: string[]): string { const list = dependencies.join(", "); const installList = dependencies.length === 1 ? dependencies[0] : ${dependencies.slice(0, -1).join(", ")} and ${dependencies.at(-1)}; return Local runtime dependency missing (${list}). No GIF/video pixels were inspected; tell the user this is an operator setup issue and the bot needs ${installList} installed before visual details are available.; }

async function inspectTurnVisualMedia({ bot, message, settings, videoInputs, shouldInspect, source }: { bot: ReplyPipelineRuntime; message: ReplyPipelineMessage; settings: Settings; videoInputs: ReplyVideoInput[]; shouldInspect: boolean; source: string; }): Promise { if (!shouldInspect || !videoInputs.length) return { imageInputs: [], promptText: "" };

const videoContextSettings = getVideoContextSettings(settings); if (!videoContextSettings.enabled || !bot.video || typeof bot.video.fetchContexts !== "function") { return { imageInputs: [], promptText: "GIF/video refs are present, but video context extraction is unavailable for this turn. Do not claim visual details from the URL or filename; say you cannot inspect it if visual specifics are needed." }; }

const maxVideos = clamp( Number(videoContextSettings.maxVideosPerMessage) || 1, 1, MAX_TURN_SCOPED_URL_VIDEOS ); const selectedVideos = videoInputs.slice(0, maxVideos); const isGifInspection = /\bgif\b/i.test(String(message.content || "")) || selectedVideos.some((video) => { const url = String(video.url || ""); const type = String(video.contentType || "").toLowerCase(); return type === "image/gif" || isLikelyAnimatedImageUrl(url) || /(?:^|.)giphy.com$/i.test(safeHostname(url)) || /(?:^|.)tenor.com$/i.test(safeHostname(url)); }); const keyframeIntervalSeconds = isGifInspection ? 1 : Number(videoContextSettings.keyframeIntervalSeconds) || 0; const maxKeyframesPerVideo = isGifInspection ? Math.max(4, Number(videoContextSettings.maxKeyframesPerVideo) || 0) : Number(videoContextSettings.maxKeyframesPerVideo) || 0; const targets = selectedVideos.map((video) => { const url = String(video.url || "").trim(); const target = bot.video.extractVideoTargets(url, 1)?.[0] || parseVideoTarget(url) || buildFallbackVideoTarget(url); return target; });

try { const result = await bot.video.fetchContexts({ targets, maxTranscriptChars: Number(videoContextSettings.maxTranscriptChars) || 1200, keyframeIntervalSeconds, maxKeyframesPerVideo, allowAsrFallback: Boolean(videoContextSettings.allowAsrFallback), maxAsrSeconds: Number(videoContextSettings.maxAsrSeconds) || 120, trace: { guildId: message.guildId, channelId: message.channelId, userId: message.author.id, source: "reply_pipeline_visual_media_inspection" } }); const videos = Array.isArray(result?.videos) ? result.videos : []; const imageInputs: ReplyImageInput[] = []; const missingDependencySet = new Set(); let keyframeErrorCount = 0; const lines = [ "GIF/video visual inspection was attempted for this visual question. Only attached keyframe images are visual evidence; metadata and error lines are not pixels." ];

selectedVideos.forEach((input, index) => {
  const ref = String(input.videoRef || `VID ${index + 1}`).trim();
  const video = videos[index] as Record<string, unknown> | undefined;
  if (!video) {
    lines.push(`${ref}: inspection returned no usable context.`);
    return;
  }
  const title = String(video.title || input.filename || "linked media").trim();
  const frameImages = Array.isArray(video.frameImages) ? video.frameImages as ReplyImageInput[] : [];
  imageInputs.push(...frameImages);
  const transcript = String(video.transcript || "").trim();
  const keyframeError = String(video.keyframeError || "").trim();
  const missingDependencies = getVideoMissingDependencies(video);
  for (const dependency of missingDependencies) missingDependencySet.add(dependency);
  if (keyframeError) keyframeErrorCount += 1;
  lines.push(`${ref}: ${title}; keyframes attached: ${frameImages.length}.`);
  if (transcript) lines.push(`${ref} transcript: ${transcript.slice(0, 500)}`);
  if (keyframeError) lines.push(`${ref} keyframe error: ${keyframeError.slice(0, 240)}`);
  if (missingDependencies.length && !frameImages.length) {
    lines.push(`${ref}: ${buildVideoDependencyFailureLine(missingDependencies)}`);
  }
});

const errorCount = Array.isArray(result?.errors) ? result.errors.length : 0;
const missingDependencies = Array.from(missingDependencySet);
if (!imageInputs.length) {
  lines.push("No keyframe pixels were attached. Do not describe visual specifics unless metadata/transcript above supports them.");
  if (missingDependencies.length) {
    lines.push(buildVideoDependencyFailureLine(missingDependencies));
  }
}
bot.store.logAction({
  kind: "runtime",
  guildId: message.guildId,
  channelId: message.channelId,
  messageId: message.id,
  userId: message.author.id,
  content: "turn_visual_media_inspection",
  metadata: {
    source,
    videoRefs: selectedVideos.map((video, index) => String(video.videoRef || `VID ${index + 1}`)),
    targetCount: targets.length,
    keyframeCount: imageInputs.length,
    keyframeErrorCount,
    missingDependencies,
    errorCount
  }
});

return {
  imageInputs,
  promptText: lines.join("

") }; } catch (error) { const errorMessage = String(error instanceof Error ? error.message : error); bot.store.logAction({ kind: "bot_error", guildId: message.guildId, channelId: message.channelId, messageId: message.id, userId: message.author.id, content: "turn_visual_media_inspection_failed", metadata: { source, error: errorMessage.slice(0, 500) } }); return { imageInputs: [], promptText: GIF/video inspection failed: ${errorMessage.slice(0, 300)}. Do not claim visual details from the URL or filename; say you cannot inspect it if visual specifics are needed. }; } }

type ReplyPipelineContext = { shouldRun: true; signal?: AbortSignal; recentMessages: ReplyRecentMessage[]; addressSignal: ReplyAddressSignal; triggerMessageIds: string[]; addressed: boolean; reactivity: number; isReplyChannel: boolean; ambientReplyEagerness: number; responseWindowEagerness: number; recentReplyWindowActive: boolean; reactionEmojiOptions: string[]; source: string; performance: ReplyPerformance; memorySlice: ReplyContinuityContext["memorySlice"]; replyMediaMemoryFacts: ReturnType<ReplyPipelineRuntime["buildMediaMemoryFacts"]>; attachmentImageInputs: ReplyImageInput[]; attachmentVideoInputs: ReplyVideoInput[]; videoLookupRefs: Record<string, string>; visualMediaContext: string; imageBudget: ReturnType<ReplyPipelineRuntime["getImageBudgetState"]>; videoBudget: ReturnType<ReplyPipelineRuntime["getVideoGenerationBudgetState"]>; mediaCapabilities: ReturnType<ReplyPipelineRuntime["getMediaGenerationCapabilities"]>; simpleImageCapabilityReady: boolean; complexImageCapabilityReady: boolean; imageCapabilityReady: boolean; videoCapabilityReady: boolean; gifBudget: ReturnType<ReplyPipelineRuntime["getGifBudgetState"]>; gifsConfigured: boolean; webSearch: ReplyWebSearchState; browserBrowse: ReturnType<ReplyPipelineRuntime["buildBrowserBrowseContext"]>; recentConversationHistory: ReplyContinuityContext["recentConversationHistory"]; recentVoiceSessionContext: Array<{ sessionId?: string | null; guildId?: string | null; channelId?: string | null; endedAt?: string | null; ageMinutes?: number | null; summaryText?: string | null; }>; memoryLookup: ReturnType<ReplyPipelineRuntime["buildMemoryLookupContext"]>; modelImageInputs: ReplyImageInput[]; imageLookup: ReturnType<ReplyPipelineRuntime["buildImageLookupContext"]>; subAgentSessions: ReturnType<ReplyPipelineRuntime["buildSubAgentSessionsRuntime"]>; replyTrace: ReplyTrace; screenShareCapability: ReturnType<ReplyPipelineRuntime["getVoiceScreenWatchCapability"]>; activeVoiceSession: ReturnType<ReplyPipelineRuntime["voiceSessionManager"]["getSession"]> | null; inVoiceChannelNow: boolean; activeVoiceParticipantRoster: string[]; musicState: ReturnType<ReplyPipelineRuntime["voiceSessionManager"]["getMusicPromptContext"]> | null; musicDisambiguation: ReturnType<ReplyPipelineRuntime["voiceSessionManager"]["getMusicDisambiguationPromptContext"]> | null; systemPrompt: string; replyPromptBase: ReplyPromptBase; initialUserPrompt: string; replyPromptCapture: ReplyPromptCaptureState; replyPrompts: ReplyPrompts; };

type ReplyIntentHandledResult = { handledByIntent: true; };

type ReplyActionableLlmResult = { handledByIntent: false; generation: ReplyGeneration; typingDelayMs: number; usedToolLoop: boolean; usedWebSearchFollowup: boolean; usedBrowserBrowseFollowup: boolean; usedMemoryLookupFollowup: boolean; usedImageLookupFollowup: boolean; mediaPromptLimit: number; replyDirective: ReplyDirective; webSearch: ReplyWebSearchState; browserBrowse: ReturnType<ReplyPipelineRuntime["buildBrowserBrowseContext"]>; memoryLookup: ReturnType<ReplyPipelineRuntime["buildMemoryLookupContext"]>; imageLookup: ReturnType<ReplyPipelineRuntime["buildImageLookupContext"]>; modelImageInputs: ReplyImageInput[]; toolResultImageInputs: ReplyImageInput[]; replyPrompts: ReplyPrompts; };

type ReplyLlmResult = ReplyIntentHandledResult | ReplyActionableLlmResult;

type ReplySkippedActionResult = { skipped: true; };

type ReplySendableActionResult = { skipped: false; reaction: ReplyReactionResult; mediaDirective: ReplyMediaDirective; finalText: string; mentionResolution: ReplyMentionResolution; screenShareOffer: ReplyScreenShareOffer; allowMediaOnlyReply: boolean; modelProducedSkip: boolean; modelProducedEmpty: boolean; payload: ReplyMessagePayload; toolImagesUsed: boolean; imageUsed: boolean; imageBudgetBlocked: boolean; imageCapabilityBlocked: boolean; imageVariantUsed: string | null; videoUsed: boolean; videoBudgetBlocked: boolean; videoCapabilityBlocked: boolean; gifUsed: boolean; gifBudgetBlocked: boolean; gifConfigBlocked: boolean; imagePrompt: ReplyDirective["imagePrompt"]; complexImagePrompt: ReplyDirective["complexImagePrompt"]; videoPrompt: ReplyDirective["videoPrompt"]; gifQuery: ReplyDirective["gifQuery"]; };

type ReplyActionResult = ReplySkippedActionResult | ReplySendableActionResult;

function isReplyActionableLlmResult(result: ReplyLlmResult): result is ReplyActionableLlmResult { return result.handledByIntent === false; }

function isReplySendableActionResult(result: ReplyActionResult): result is ReplySendableActionResult { return result.skipped === false; }

const TOOL_NARRATION_PATTERN = /^(?:I\s+(?:searched|looked|found|checked|browsed|fetched|retrieved|queried|ran|used|called|executed|performed|attempted|tried)\b|(?:After|Based on|From)\s+(?:searching|looking|browsing|checking|running|using|calling)\b)/i;

function looksLikeToolNarration(text: string): boolean { const trimmed = text.trim(); if (!trimmed) return false; return TOOL_NARRATION_PATTERN.test(trimmed); }

function logReplyPipelineGate( bot: ReplyPipelineRuntime, { message, settings, options, allow, reason, sendBudgetAllowed = null, talkNowAllowed = null, ctx = null }: { message: ReplyPipelineMessage; settings: Settings; options: ReplyAttemptOptions; allow: boolean; reason: string; sendBudgetAllowed?: boolean | null; talkNowAllowed?: boolean | null; ctx?: ReplyPipelineContext | null; } ) { const triggerMessageIds = [ ...new Set( [...(Array.isArray(options.triggerMessageIds) ? options.triggerMessageIds : []), message.id] .map((value) => String(value || "").trim()) .filter(Boolean) ) ]; const source = String(options.source || "message_event").trim() || "message_event"; const triggerMessageId = triggerMessageIds[0] || String(message.id || "").trim() || null; const isReplyChannel = bot.isReplyChannel(settings, message.channelId); const addressSignal = options.addressSignal && typeof options.addressSignal === "object" ? options.addressSignal : null;

bot.store.logAction({ kind: "text_runtime", guildId: message.guildId, channelId: message.channelId, messageId: message.id, userId: message.author?.id || null, content: "reply_pipeline_gate", metadata: { ...buildRuntimeDecisionCorrelation({ botId: bot.client.user?.id || null, triggerMessageId, source, stage: "pipeline", allow, reason }), triggerMessageIds, forceRespond: Boolean(options.forceRespond), forceDecisionLoop: Boolean(options.forceDecisionLoop), isReplyChannel, sendBudgetAllowed, talkNowAllowed, ctxBuilt: Boolean(ctx), ctxShouldRun: ctx ? Boolean(ctx.shouldRun) : null, addressed: ctx ? Boolean(ctx.addressed) : null, addressSignal: addressSignal ? { direct: Boolean(addressSignal.direct), inferred: Boolean(addressSignal.inferred), triggered: Boolean(addressSignal.triggered), reason: String(addressSignal.reason || "llm_decides"), confidence: Math.max(0, Math.min(1, Number(addressSignal.confidence) || 0)), threshold: Math.max(0.4, Math.min(0.95, Number(addressSignal.threshold) || 0.62)), confidenceSource: String(addressSignal.confidenceSource || "fallback") } : null } }); }

function buildReplyToolAvailabilityState( settings: Settings, { webSearch, browserBrowse, imageLookup, userId }: Pick<ReplyPipelineContext, "webSearch" | "browserBrowse" | "imageLookup"> & { userId: string | null | undefined } ): { tools: ReplyToolDefinition[]; capabilities: { webSearchAvailable?: boolean; webScrapeAvailable?: boolean; browserBrowseAvailable?: boolean; memoryAvailable?: boolean; imageLookupAvailable?: boolean; swarmToolsAvailable?: boolean; voiceToolsAvailable?: boolean; }; includedTools: string[]; excludedTools: Array<{ name: string; reason: string }>; } { const memoryEnabled = Boolean(getMemorySettings(settings).enabled); const voiceEnabled = Boolean(getVoiceSettings(settings).enabled); const codeAgentEnabled = isDevTaskEnabled(settings) && isDevTaskUserAllowed(settings, userId); const minecraftEnabled = isMinecraftEnabled(settings);

const webSearchReason = !webSearch?.enabled ? "settings_disabled" : !webSearch?.configured ? "provider_unconfigured" : webSearch?.optedOutByUser ? "opted_out_by_user" : webSearch?.blockedByBudget ? "budget_blocked" : webSearch?.budget?.canSearch === false ? "budget_exhausted" : "available"; const browserBrowseReason = !browserBrowse?.enabled ? "settings_disabled" : !browserBrowse?.configured ? "runtime_unavailable" : browserBrowse?.blockedByBudget ? "budget_blocked" : browserBrowse?.budget?.canBrowse === false ? "budget_exhausted" : "available"; const imageLookupReason = imageLookup?.enabled ? "available" : imageLookup?.error ? "lookup_error" : "no_history_images"; const memoryReason = memoryEnabled ? "available" : "settings_disabled"; const swarmToolsReason = codeAgentEnabled ? "available" : "settings_disabled"; const voiceToolReason = voiceEnabled ? "available" : "settings_disabled";

const videoContextEnabled = Boolean(getVideoContextSettings(settings).enabled); const videoContextReason = videoContextEnabled ? "available" : "settings_disabled";

const capabilities = { webSearchAvailable: webSearchReason === "available", webScrapeAvailable: webSearchReason === "available", browserBrowseAvailable: browserBrowseReason === "available", memoryAvailable: memoryReason === "available", imageLookupAvailable: imageLookupReason === "available", videoContextAvailable: videoContextReason === "available", swarmToolsAvailable: swarmToolsReason === "available", voiceToolsAvailable: voiceToolReason === "available", minecraftAvailable: minecraftEnabled }; const tools = buildReplyToolSet(settings, capabilities); const includedSet = new Set(tools.map((tool) => String(tool.name || "").trim()).filter(Boolean)); const candidates: Array<{ name: string; reason: string }> = [ { name: "web_search", reason: webSearchReason }, { name: "web_scrape", reason: webSearchReason }, { name: "video_context", reason: videoContextReason }, { name: "browser_browse", reason: browserBrowseReason }, { name: "memory_search", reason: memoryReason }, { name: "memory_write", reason: memoryReason }, { name: "conversation_search", reason: "available" }, { name: "image_lookup", reason: imageLookupReason }, { name: "spawn_code_worker", reason: swarmToolsReason }, ...SWARM_TOOL_SCHEMAS.map((schema) => ({ name: schema.name, reason: swarmToolsReason })), { name: "minecraft_task", reason: minecraftEnabled ? "available" : "settings_disabled" }, ...VOICE_TOOL_SCHEMAS.map((schema) => ({ name: schema.name, reason: voiceToolReason })) ];

return { tools, capabilities, includedTools: candidates .map((candidate) => candidate.name) .filter((name, index, values) => values.indexOf(name) === index && includedSet.has(name)), excludedTools: candidates .filter((candidate, index, values) => values.findIndex((entry) => entry.name === candidate.name) === index && !includedSet.has(candidate.name) ) .map((candidate) => ({ name: candidate.name, reason: candidate.reason })) }; }

function logReplyToolAvailability( bot: ReplyPipelineRuntime, { message, options, includedTools, excludedTools, capabilities }: { message: ReplyPipelineMessage; options: ReplyAttemptOptions; includedTools: string[]; excludedTools: Array<{ name: string; reason: string }>; capabilities: { webSearchAvailable?: boolean; webScrapeAvailable?: boolean; browserBrowseAvailable?: boolean; memoryAvailable?: boolean; imageLookupAvailable?: boolean; swarmToolsAvailable?: boolean; voiceToolsAvailable?: boolean; }; } ) { const triggerMessageIds = [ ...new Set( [...(Array.isArray(options.triggerMessageIds) ? options.triggerMessageIds : []), message.id] .map((value) => String(value || "").trim()) .filter(Boolean) ) ]; const source = String(options.source || "message_event").trim() || "message_event"; const triggerMessageId = triggerMessageIds[0] || String(message.id || "").trim() || null; bot.store.logAction({ kind: "text_runtime", guildId: message.guildId, channelId: message.channelId, messageId: message.id, userId: message.author?.id || null, content: "reply_tool_availability", metadata: { ...buildRuntimeDecisionCorrelation({ botId: bot.client.user?.id || null, triggerMessageId, source, stage: "tool_availability", allow: includedTools.length > 0, reason: includedTools.length > 0 ? "tools_available" : "no_tools_available" }), triggerMessageIds, includedToolCount: includedTools.length, excludedToolCount: excludedTools.length, includedTools, excludedTools, capabilities } }); }

async function buildReplyContext( bot: ReplyPipelineRuntime, message: ReplyPipelineMessage, settings: Settings, options: ReplyAttemptOptions ): Promise<ReplyPipelineContext | false> { const memorySettings = getMemorySettings(settings); const activity = getActivitySettings(settings); const automationsSettings = getAutomationsSettings(settings); const discovery = getDiscoverySettings(settings); const voiceSettings = getVoiceSettings(settings); const visionSettings = getVisionSettings(settings); const recentMessages = Array.isArray(options.recentMessages) ? options.recentMessages : bot.store.getRecentMessages(message.channelId, memorySettings.promptSlice.maxRecentMessages); const addressSignal = options.addressSignal || await bot.getReplyAddressSignal(settings, message, recentMessages); const triggerMessageIds = [ ...new Set( [...(Array.isArray(options.triggerMessageIds) ? options.triggerMessageIds : []), message.id] .map((value) => String(value || "").trim()) .filter(Boolean) ) ]; const addressed = Boolean(addressSignal?.triggered); const reactivity = clamp(Number(activity.reactivity) || 0, 0, 100); const isReplyChannel = bot.isReplyChannel(settings, message.channelId); const ambientReplyEagerness = clamp( Number(activity.ambientReplyEagerness) || 0, 0, 100 ); const responseWindowEagerness = clamp( Number(activity.responseWindowEagerness) || 0, 0, 100 ); const textAttentionState = resolveTextAttentionState({ botUserId: bot.client.user?.id || null, settings, message, recentMessages, addressSignal, triggerMessageId: message.id, triggerAuthorId: message.author?.id || null, triggerReferenceMessageId: message.reference?.messageId || message.referencedMessage?.id || null }); const recentReplyWindowActive = textAttentionState.recentReplyWindowActive; const reactionEmojiOptions = [ ...new Set([...bot.getReactionEmojiOptions(message.guild), ...UNICODE_REACTIONS]) ];

const shouldRunDecisionLoop = bot.shouldAttemptReplyDecision({ settings, message, recentMessages, addressSignal, isReplyChannel, forceRespond: Boolean(options.forceRespond), forceDecisionLoop: Boolean(options.forceDecisionLoop), triggerMessageId: message.id, triggerAuthorId: message.author?.id || null, triggerReferenceMessageId: message.reference?.messageId || message.referencedMessage?.id || null }); if (!shouldRunDecisionLoop) return false;

const source = String(options.source || "message_event"); const performance = createReplyPerformanceTracker({ messageCreatedAtMs: message?.createdTimestamp, source, seed: options.performanceSeed });

const memorySliceStartedAtMs = Date.now(); const continuity = await loadConversationContinuityContext({ settings, userId: message.author.id, guildId: message.guildId, channelId: message.channelId, queryText: message.content, trace: { guildId: message.guildId, channelId: message.channelId, userId: message.author.id }, source, recentMessages, loadFactProfile: (payload) => bot.loadFactProfile({ settings: payload.settings, userId: payload.userId, guildId: payload.guildId ?? message.guildId ?? null, channelId: payload.channelId, queryText: payload.queryText, trace: payload.trace, source: payload.source }), loadRecentConversationHistory: (payload) => bot.getConversationHistoryForPrompt(payload) }); const memorySlice = continuity.memorySlice; const behavioralFacts = await loadBehavioralMemoryFacts(bot, { settings, guildId: message.guildId, channelId: message.channelId, queryText: message.content, participantIds: Array.isArray(memorySlice?.participantProfiles) ? memorySlice.participantProfiles .map((entry) => String((entry as Record<string, unknown>)?.userId || "").trim()) .filter(Boolean) : [], trace: { guildId: message.guildId, channelId: message.channelId, userId: message.author.id, source: "reply_pipeline_behavioral_memory" }, limit: 8 }); performance.memorySliceMs = Math.max(0, Date.now() - memorySliceStartedAtMs); const replyMediaMemoryFacts = bot.buildMediaMemoryFacts({ userFacts: memorySlice.userFacts, relevantFacts: memorySlice.relevantFacts }); const directAttachmentImageInputs: ReplyImageInput[] = bot.getImageInputs(message); const directAttachmentVideoInputs: ReplyVideoInput[] = bot.getVideoInputs(message) || []; const turnScopedUrlMedia = collectTurnScopedUrlMediaInputs({ message, recentMessages, triggerMessageIds, existingImageUrls: directAttachmentImageInputs.map((image) => String(image?.url || "").trim()), existingVideoUrls: directAttachmentVideoInputs.map((video) => String(video?.url || "").trim()) }); const attachmentImageInputs: ReplyImageInput[] = bot.mergeImageInputs({ baseInputs: directAttachmentImageInputs, extraInputs: turnScopedUrlMedia.imageInputs, maxInputs: MAX_MODEL_IMAGE_INPUTS }); const attachmentVideoInputs: ReplyVideoInput[] = mergeVideoInputs([ ...directAttachmentVideoInputs, ...turnScopedUrlMedia.videoInputs ]) .map((video, index) => ({ ...video, videoRef: VID ${index + 1} })); const videoLookupRefs = Object.fromEntries( attachmentVideoInputs .map((video) => [ String(video.videoRef || "").trim(), String(video.url || "").trim() ]) .filter(([videoRef, url]) => Boolean(videoRef && url)) ); const imageBudget = bot.getImageBudgetState(settings); const videoBudget = bot.getVideoGenerationBudgetState(settings); const mediaCapabilities = bot.getMediaGenerationCapabilities(settings); const simpleImageCapabilityReady = mediaCapabilities.simpleImageReady; const complexImageCapabilityReady = mediaCapabilities.complexImageReady; const imageCapabilityReady = simpleImageCapabilityReady || complexImageCapabilityReady; const videoCapabilityReady = mediaCapabilities.videoReady; const gifBudget = bot.getGifBudgetState(settings); const gifsConfigured = Boolean(bot.gifs?.isConfigured?.()); const webSearch: ReplyWebSearchState = bot.buildWebSearchContext(settings, message.content); const browserBrowse = bot.buildBrowserBrowseContext(settings); const recentConversationHistory = continuity.recentConversationHistory; const recentVoiceSessionContext = memorySettings.enabled && typeof bot.memory?.getRecentVoiceSessionSummariesForPrompt === "function" ? bot.memory.getRecentVoiceSessionSummariesForPrompt({ guildId: message.guildId, channelId: message.channelId, referenceAtMs: message.createdTimestamp }) : []; const memoryLookup = bot.buildMemoryLookupContext({ settings }); const visualMediaContext = await inspectTurnVisualMedia({ bot, message, settings, videoInputs: attachmentVideoInputs, shouldInspect: isVisualMediaInspectionRequest(message.content), source }); const modelImageInputs: ReplyImageInput[] = bot.mergeImageInputs({ baseInputs: attachmentImageInputs, extraInputs: visualMediaContext.imageInputs, maxInputs: MAX_MODEL_IMAGE_INPUTS }); const imageLookup = bot.buildImageLookupContext({ recentMessages, excludedUrls: modelImageInputs.map((image) => String(image?.url || "").trim()) });

if (Boolean(visionSettings.enabled) && imageLookup.candidates?.length) { // Fire-and-forget: caption uncaptioned images in background for future text matching. // Historical images stay out of direct model vision context until the model explicitly asks. bot.captionRecentHistoryImages({ candidates: imageLookup.candidates, settings, trace: { guildId: message.guildId, channelId: message.channelId, userId: message.author.id, source: "reply_pipeline_auto_caption" } }); } const replyTrace = { guildId: message.guildId, channelId: message.channelId, userId: message.author.id, source, event: null, reason: null, messageId: message.id }; const screenShareCapability = bot.getVoiceScreenWatchCapability({ settings, guildId: message.guildId, channelId: message.channelId, requesterUserId: message.author?.id || null }); const activeVoiceSession = typeof bot.voiceSessionManager?.getSession === "function" ? bot.voiceSessionManager.getSession(message.guildId) : null; const inVoiceChannelNow = Boolean(activeVoiceSession && !activeVoiceSession.ending); const activeVoiceParticipantRoster = inVoiceChannelNow && typeof bot.voiceSessionManager?.getVoiceChannelParticipants === "function" ? bot.voiceSessionManager .getVoiceChannelParticipants(activeVoiceSession) .map((entry) => String(entry?.displayName || "").trim()) .filter(Boolean) : []; const musicDisambiguation = inVoiceChannelNow && typeof bot.voiceSessionManager?.getMusicDisambiguationPromptContext === "function" ? bot.voiceSessionManager.getMusicDisambiguationPromptContext(activeVoiceSession) : null; const musicState = inVoiceChannelNow && typeof bot.voiceSessionManager?.getMusicPromptContext === "function" ? bot.voiceSessionManager.getMusicPromptContext(activeVoiceSession) : null; const subAgentSessions = bot.buildSubAgentSessionsRuntime(); const activeMinecraftSession = findReusableMinecraftSession(subAgentSessions.manager, { ownerUserId: message.author.id, scopeKey: buildMinecraftSessionScopeKey({ guildId: message.guildId, channelId: message.channelId }) }); const minecraftSessionHint = getMinecraftSessionPromptHint(activeMinecraftSession);

const systemPrompt = buildSystemPrompt(settings); const replyPromptBase: ReplyPromptBase = { message: { authorName: message.member?.displayName || message.author.username, content: message.content }, triggerMessageIds, imageInputs: modelImageInputs, recentMessages, participantProfiles: memorySlice.participantProfiles, selfFacts: memorySlice.selfFacts, loreFacts: memorySlice.loreFacts, guidanceFacts: Array.isArray(memorySlice.guidanceFacts) ? memorySlice.guidanceFacts : [], behavioralFacts, userFacts: memorySlice.userFacts, relevantFacts: memorySlice.relevantFacts, emojiHints: bot.getEmojiHints(message.guild), reactionEmojiOptions, allowReplySimpleImages: discovery.allowReplyImages && simpleImageCapabilityReady && imageBudget.canGenerate, allowReplyComplexImages: discovery.allowReplyImages && complexImageCapabilityReady && imageBudget.canGenerate, remainingReplyImages: imageBudget.remaining, allowReplyVideos: discovery.allowReplyVideos && videoCapabilityReady && videoBudget.canGenerate, remainingReplyVideos: videoBudget.remaining, allowReplyGifs: discovery.allowReplyGifs && gifsConfigured && gifBudget.canFetch, remainingReplyGifs: gifBudget.remaining, gifRepliesEnabled: discovery.allowReplyGifs, gifsConfigured, ambientReplyEagerness, responseWindowEagerness, recentReplyWindowActive, textAttentionMode: textAttentionState.mode, textAttentionReason: textAttentionState.reason, reactivity, addressing: { directlyAddressed: addressed, directAddressConfidence: Number(addressSignal?.confidence) || 0, directAddressThreshold: Number(addressSignal?.threshold) || 0.62, mentionsOtherUsers: Boolean( !addressed && message.mentions?.users?.size > 0 && !message.mentions.users.has(bot.client.user?.id) ), repliesToOtherUser: Boolean( !addressed && message.mentions?.repliedUser && message.mentions.repliedUser.id !== bot.client.user?.id ) }, allowMemoryDirective: memorySettings.enabled, allowAutomationDirective: Boolean(automationsSettings.enabled), automationTimeZoneLabel: getLocalTimeZoneLabel(), voiceMode: { enabled: Boolean(voiceSettings.enabled), activeSession: inVoiceChannelNow, participantRoster: activeVoiceParticipantRoster, musicState, musicDisambiguation }, recentConversationHistory, recentVoiceSessionContext, minecraftSessionHint, screenShare: screenShareCapability, visualMediaContext: visualMediaContext.promptText, channelMode: isReplyChannel ? "reply_channel" : bot.isDiscoveryChannel(settings, message.channelId) ? "discovery_channel" : "other_channel", maxMediaPromptChars: resolveMaxMediaPromptLen(settings), mediaPromptCraftGuidance: getMediaPromptCraftGuidance(settings) }; const initialUserPrompt = buildReplyPrompt({ ...replyPromptBase, imageInputs: modelImageInputs, videoInputs: attachmentVideoInputs, webSearch, browserBrowse, memoryLookup, imageLookup }); const replyPromptCapture = createReplyPromptCapture({ systemPrompt, initialUserPrompt }); const replyPrompts = buildLoggedReplyPrompts(replyPromptCapture, 0);

return { shouldRun: true, recentMessages, addressSignal, triggerMessageIds, addressed, reactivity, isReplyChannel, ambientReplyEagerness, responseWindowEagerness, recentReplyWindowActive, reactionEmojiOptions, source, performance, memorySlice, replyMediaMemoryFacts, attachmentImageInputs, attachmentVideoInputs, videoLookupRefs, visualMediaContext: visualMediaContext.promptText, imageBudget, videoBudget, mediaCapabilities, simpleImageCapabilityReady, complexImageCapabilityReady, imageCapabilityReady, videoCapabilityReady, gifBudget, gifsConfigured, webSearch, browserBrowse, recentConversationHistory, recentVoiceSessionContext, memoryLookup, modelImageInputs, imageLookup, subAgentSessions, replyTrace, screenShareCapability, activeVoiceSession, inVoiceChannelNow, activeVoiceParticipantRoster, musicState, musicDisambiguation, systemPrompt, replyPromptBase, initialUserPrompt, replyPromptCapture, replyPrompts }; }

async function executeReplyLlm( bot: ReplyPipelineRuntime, message: ReplyPipelineMessage, settings: Settings, options: ReplyAttemptOptions, ctx: ReplyPipelineContext ): Promise { const { addressSignal, triggerMessageIds, source, performance, signal, replyTrace, systemPrompt, initialUserPrompt, replyPromptCapture, activeVoiceSession, inVoiceChannelNow, videoLookupRefs } = ctx; const { memoryLookup } = ctx; let { webSearch, browserBrowse, modelImageInputs, imageLookup, replyPrompts } = ctx; const { subAgentSessions } = ctx;

const replyToolAvailability = buildReplyToolAvailabilityState(settings, { webSearch, browserBrowse, imageLookup, userId: message.author.id }); const replyTools = replyToolAvailability.tools; logReplyToolAvailability(bot, { message, options, includedTools: replyToolAvailability.includedTools, excludedTools: replyToolAvailability.excludedTools, capabilities: replyToolAvailability.capabilities });

const activeVoiceCallbacks = inVoiceChannelNow && activeVoiceSession ? bot.voiceSessionManager.buildVoiceToolCallbacks({ session: activeVoiceSession, settings }) : null;

const replyToolRuntime: ReplyToolRuntime = { search: bot.search, video: bot.video ? { fetchContext: async ({ url, settings: toolSettings, trace }) => { const videoContextSettings = getVideoContextSettings(toolSettings); const targets = bot.video.extractVideoTargets(url, 1); if (!targets.length) { // URL didn't match a known video host — fall through with a generic target targets.push({ key: generic:${url}, url, kind: "generic", provider: "generic", videoId: null }); } const result = await bot.video.fetchContexts({ targets, maxTranscriptChars: Number(videoContextSettings.maxTranscriptChars) || 1200, keyframeIntervalSeconds: Number(videoContextSettings.keyframeIntervalSeconds) || 0, maxKeyframesPerVideo: Number(videoContextSettings.maxKeyframesPerVideo) || 0, allowAsrFallback: Boolean(videoContextSettings.allowAsrFallback), maxAsrSeconds: Number(videoContextSettings.maxAsrSeconds) || 120, trace }); if (result.errors?.length && !result.videos?.length) { return { text: Video context extraction failed for ${url}: ${result.errors[0]?.error || "unknown error"}. Try web_scrape or browser_browse as fallback., isError: true }; } const video = result.videos?.[0]; if (!video) { return { text: No video metadata could be extracted from ${url}. Try web_scrape or browser_browse instead., isError: true }; } const lines: string[] = []; lines.push(Provider: ${video.provider || "unknown"}); lines.push(Title: ${video.title || "untitled"}); lines.push(Channel: ${video.channel || "unknown"}); if (video.url) lines.push(URL: ${video.url}); if (video.publishedAt) lines.push(Published: ${video.publishedAt}); if (video.durationSeconds) lines.push(Duration: ${video.durationSeconds}s); if (video.viewCount) lines.push(Views: ${video.viewCount}); if (video.description) lines.push(Description: ${video.description}); if (video.transcript) lines.push(Transcript (${video.transcriptSource || "unknown source"}): ${video.transcript}); if (video.transcriptError) lines.push(Transcript error: ${video.transcriptError}); if (video.keyframeError) lines.push(Keyframe error: ${video.keyframeError}); const frameImages = video.frameImages || []; const missingDependencies = getVideoMissingDependencies(video as Record<string, unknown>); if (missingDependencies.length && !frameImages.length) { lines.push(buildVideoDependencyFailureLine(missingDependencies)); } if (frameImages.length) lines.push(Keyframes: ${frameImages.length} frame(s) attached); const toolResultText = lines.join(" "); const frameImageBytes = frameImages.reduce((sum, img) => sum + (img.dataBase64 ? Math.ceil(img.dataBase64.length * 3 / 4) : 0), 0); bot.store.logAction({ kind: "runtime", guildId: trace.guildId, channelId: trace.channelId, userId: trace.userId, content: "video_context_tool_result", metadata: { url, provider: video.provider || "unknown", title: video.title || "untitled", durationSeconds: video.durationSeconds || null, hasTranscript: Boolean(video.transcript), transcriptChars: video.transcript?.length || 0, keyframeCount: frameImages.length, keyframePayloadBytes: frameImageBytes, keyframeError: video.keyframeError || null, keyframeErrorCode: video.keyframeErrorCode || null, transcriptErrorCode: video.transcriptErrorCode || null, missingDependencies, toolResultChars: toolResultText.length, toolResultPreview: toolResultText.slice(0, 300) } }); return { text: toolResultText, imageInputs: frameImages.length ? frameImages : undefined }; } } : undefined, browser: { browse: async ({ settings: toolSettings, query, guildId, channelId, userId, source }) => { browserBrowse = await bot.runModelRequestedBrowserBrowse({ settings: toolSettings, browserBrowse, query, guildId, channelId, userId, source, signal }); return browserBrowse; } }, memory: bot.memory, store: bot.store, swarm: { peerManager: bot.swarmPeerManager, reservationKeeper: bot.swarmReservationKeeper, activityBridge: bot.swarmActivityBridge }, subAgentSessions, voiceSession: activeVoiceCallbacks || undefined, voiceJoin: Boolean(getVoiceSettings(settings).enabled) && bot.voiceSessionManager ? async () => { try { const joined = await bot.voiceSessionManager.requestJoin({ message, settings, intentConfidence: 1 }); if (!joined) { return { ok: false, reason: "join_not_handled" }; } const session = bot.voiceSessionManager.getSession(message.guildId); if (!session || session.ending) { return { ok: false, reason: "session_not_available_after_join" }; } const callbacks = bot.voiceSessionManager.buildVoiceToolCallbacks({ session, settings }); const voiceChannelId = String(session.voiceChannelId || ""); const guild = bot.client.guilds?.cache?.get(message.guildId); const voiceChannel = voiceChannelId && guild?.channels?.cache?.get(voiceChannelId); const voiceChannelName = String(voiceChannel?.name || voiceChannelId || "voice channel"); return { ok: true, voiceSession: callbacks, voiceChannelName }; } catch (error) { return { ok: false, reason: String((error as Error)?.message || error) }; } } : undefined }; const replyToolContext: ReplyToolContext = { settings, guildId: message.guildId, channelId: message.channelId, userId: message.author.id, sourceMessageId: message.id, sourceText: message.content, botUserId: bot.client.user?.id || undefined, actorName: message.member?.displayName || message.author?.username || undefined, trace: { ...replyTrace, source }, videoLookup: { refs: videoLookupRefs }, signal }; let replyContextMessages: ContextMessage[] = [];

throwIfAborted(signal, "Reply cancelled"); const typingStartedAtMs = Date.now(); await message.channel.sendTyping(); const typingDelayMs = Math.max(0, Date.now() - typingStartedAtMs); const llm1StartedAtMs = Date.now(); let generation: ReplyGeneration = await bot.llm.generate({ settings, systemPrompt, userPrompt: initialUserPrompt, imageInputs: modelImageInputs, contextMessages: replyContextMessages, jsonSchema: REPLY_OUTPUT_JSON_SCHEMA, tools: replyTools, trace: replyTrace, signal }); performance.llm1Ms = Math.max(0, Date.now() - llm1StartedAtMs); let usedWebSearchFollowup = false; let usedBrowserBrowseFollowup = false; let usedMemoryLookupFollowup = false; let usedImageLookupFollowup = false; let toolResultImageInputs: ReplyImageInput[] = []; const REPLY_TOOL_LOOP_MAX_STEPS = 2; const REPLY_TOOL_LOOP_MAX_CALLS = 3; let replyToolLoopSteps = 0; let replyTotalToolCalls = 0; const followupStartedAtMs = Date.now();

while ( generation.toolCalls?.length > 0 && replyToolLoopSteps < REPLY_TOOL_LOOP_MAX_STEPS && replyTotalToolCalls < REPLY_TOOL_LOOP_MAX_CALLS ) { throwIfAborted(signal, "Reply cancelled"); const assistantContent = buildContextContentBlocks(generation.rawContent, generation.text); // On the first iteration, seed the context with the original user prompt. // On subsequent iterations the prompt is already in the history — don't duplicate it. if (replyContextMessages.length === 0) { replyContextMessages = [ { role: "user", content: initialUserPrompt }, { role: "assistant", content: assistantContent } ]; } else { replyContextMessages = [ ...replyContextMessages, { role: "assistant", content: assistantContent } ]; }

const toolResultMessages: ContentBlock[] = [];
let toolResultImageInputsAdded = false;
const mergeToolResultImages = (extraInputs: ReplyImageInput[] | undefined) => {
  if (!Array.isArray(extraInputs) || extraInputs.length === 0 || typeof bot.mergeImageInputs !== "function") {
    return false;
  }
  modelImageInputs = bot.mergeImageInputs({
    baseInputs: modelImageInputs,
    extraInputs,
    maxInputs: MAX_MODEL_IMAGE_INPUTS
  });
  toolResultImageInputs = bot.mergeImageInputs({
    baseInputs: toolResultImageInputs,
    extraInputs,
    maxInputs: MAX_MODEL_IMAGE_INPUTS
  });
  return true;
};

// Separate sub-agent tools (can run concurrently) from sequential tools
const CONCURRENT_TOOL_NAMES = new Set(["spawn_code_worker", "browser_browse"]);
const eligibleToolCalls = generation.toolCalls.slice(
  0,
  Math.max(0, REPLY_TOOL_LOOP_MAX_CALLS - replyTotalToolCalls)
);
replyTotalToolCalls += eligibleToolCalls.length;

const concurrentCalls = eligibleToolCalls.filter((tc) => CONCURRENT_TOOL_NAMES.has(tc.name));
const sequentialCalls = eligibleToolCalls.filter((tc) => !CONCURRENT_TOOL_NAMES.has(tc.name));

// Run concurrent sub-agent calls in parallel
const concurrentResults = new Map<string, ReplyToolExecutionResult>();
if (concurrentCalls.length > 0) {
  const settledCalls = await Promise.allSettled(concurrentCalls.map(async (toolCall) => {
    throwIfAborted(signal, "Reply cancelled");
    const toolInput = toolCall.input;
    const result = await executeReplyTool(toolCall.name, toolInput, replyToolRuntime, replyToolContext);
    if (mergeToolResultImages(result?.imageInputs)) {
      toolResultImageInputsAdded = true;
    }
    concurrentResults.set(toolCall.id, result);
  }));
  settledCalls.forEach((settled, index) => {
    if (settled.status === "fulfilled") return;
    const toolCall = concurrentCalls[index];
    const errorMessage =
      settled.reason instanceof Error
        ? settled.reason.message
        : String(settled.reason || "unknown_error");
    concurrentResults.set(toolCall.id, {
      content: `${toolCall.name} failed: ${errorMessage}`,
      isError: true
    });
    bot.store?.logAction?.({
      kind: "bot_error",
      guildId: message.guildId,
      channelId: message.channelId,
      messageId: message.id,
      userId: message.author?.id || null,
      content: `reply_tool_concurrent_failure:${toolCall.name}`,
      metadata: {
        source,
        toolCallId: toolCall.id,
        error: errorMessage
      }
    });
  });
}

// Run sequential tools in order
const sequentialResults = new Map<string, ReplyToolExecutionResult>();
for (const toolCall of sequentialCalls) {
  throwIfAborted(signal, "Reply cancelled");
  const toolInput = toolCall.input;
  let result: ReplyToolExecutionResult;
  if (toolCall.name === "web_search") {
    const toolQuery = String(toolInput.query || "");
    webSearch = await runModelRequestedWebSearch(
      { llm: bot.llm, search: bot.search, memory: bot.memory },
      {
        settings,
        webSearch,
        query: toolQuery,
        trace: {
          ...replyTrace,
          source
        },
        signal
      }
    );
    usedWebSearchFollowup = Boolean(webSearch?.used);
    const rows = Array.isArray(webSearch?.results) ? webSearch.results : [];
    result = {
      isError: Boolean(webSearch?.error),
      content: webSearch?.error
        ? `Web search failed: ${String(webSearch.error)}`
        : rows.length || String(webSearch?.summaryText || "").trim()
          ? `Web results for "${String(webSearch?.query || toolQuery)}":

${[ String(webSearch?.summaryText || "").trim() ? Summary: ${String(webSearch?.summaryText || "").trim()} : "", rows .map((item, index) => { const title = String(item?.title || "untitled").trim(); const url = String(item?.url || "").trim(); const domain = String(item?.domain || "").trim(); const snippet = String(item?.snippet || "").trim(); const pageSummary = String(item?.pageSummary || "").trim(); const domainLabel = domain ? (${domain}) : ""; const snippetLine = snippet ? Snippet: ${snippet} : ""; const pageLine = pageSummary ? Page: ${pageSummary} : ""; return [${index + 1}] ${title}${domainLabel} URL: ${url}${snippetLine}${pageLine}; }) .join("

") ] .filter(Boolean) .join("

")} :No results found for: "${toolQuery}"` }; } else { result = await executeReplyTool( toolCall.name, toolInput, replyToolRuntime, replyToolContext ); }

  if (mergeToolResultImages(result?.imageInputs)) {
    toolResultImageInputsAdded = true;
  }

  if (toolCall.name === "memory_search" && !result.isError) {
    usedMemoryLookupFollowup = true;
  } else if (toolCall.name === "image_lookup" && !result.isError) {
    const imageLookupRequest = String(toolInput.imageId || toolInput.query || "");
    imageLookup = await bot.runModelRequestedImageLookup({
      imageLookup,
      query: imageLookupRequest
    });
    if (mergeToolResultImages(imageLookup.selectedImageInputs || [])) {
      toolResultImageInputsAdded = true;
    }
    usedImageLookupFollowup = Boolean(imageLookup?.used);
  }

  sequentialResults.set(toolCall.id, result);
}

// Collect results in original order
for (const toolCall of eligibleToolCalls) {
  const result = concurrentResults.get(toolCall.id) || sequentialResults.get(toolCall.id);
  if (result) {
    if (toolCall.name === "browser_browse" && !result.isError) {
      usedBrowserBrowseFollowup = Boolean(browserBrowse?.used);
    }
    toolResultMessages.push({
      type: "tool_result",
      tool_use_id: toolCall.id,
      content: result.content
    });
  }
}

replyContextMessages = [
  ...replyContextMessages,
  { role: "user", content: toolResultMessages }
];

throwIfAborted(signal, "Reply cancelled");
const toolLoopUserPrompt = toolResultImageInputsAdded
  ? "Attached are images returned by the previous tool call. Use them if they help. If you want to include those exact images in the final Discord reply, set media to {\"type\":\"tool_images\",\"prompt\":null}."
  : "";
appendReplyFollowupPrompt(replyPromptCapture, toolLoopUserPrompt);
generation = await bot.llm.generate({
  settings,
  systemPrompt,
  userPrompt: toolLoopUserPrompt,
  imageInputs: modelImageInputs,
  contextMessages: replyContextMessages,
  jsonSchema: REPLY_OUTPUT_JSON_SCHEMA,
  tools: replyTools,
  trace: {
    ...replyTrace,
    event: `reply_tool_loop:${replyToolLoopSteps + 1}`
  },
  signal
});
replyToolLoopSteps += 1;

}

const mediaPromptLimit = resolveMaxMediaPromptLen(settings); const replyDirective = parseStructuredReplyOutput(generation.text, mediaPromptLimit); replyPrompts = buildLoggedReplyPrompts(replyPromptCapture, replyToolLoopSteps);

const automationIntentHandled = await bot.maybeHandleStructuredAutomationIntent({ message, settings, replyDirective, generation, source, triggerMessageIds, addressing: addressSignal, performance, replyPrompts }); if (automationIntentHandled) return { handledByIntent: true };

if ( replyToolLoopSteps > 0 || usedWebSearchFollowup || usedBrowserBrowseFollowup || usedMemoryLookupFollowup || usedImageLookupFollowup ) { performance.followupMs = Math.max(0, Date.now() - followupStartedAtMs); }

return { handledByIntent: false, generation, typingDelayMs, usedToolLoop: replyToolLoopSteps > 0, usedWebSearchFollowup, usedBrowserBrowseFollowup, usedMemoryLookupFollowup, usedImageLookupFollowup, mediaPromptLimit, replyDirective, webSearch, browserBrowse, memoryLookup, imageLookup, modelImageInputs, toolResultImageInputs, replyPrompts }; }

async function dispatchReplyActions( bot: ReplyPipelineRuntime, message: ReplyPipelineMessage, settings: Settings, _options: ReplyAttemptOptions, ctx: ReplyPipelineContext, llmResult: ReplyActionableLlmResult ): Promise { const discovery = getDiscoverySettings(settings); const { addressSignal, triggerMessageIds, reactionEmojiOptions, source, performance, replyMediaMemoryFacts } = ctx; const { generation, usedToolLoop, usedWebSearchFollowup, mediaPromptLimit, replyDirective, webSearch, toolResultImageInputs, replyPrompts } = llmResult; if (replyDirective.parseState === "unstructured") { if (usedToolLoop && looksLikeToolNarration(String(generation.text || ""))) { bot.logSkippedReply({ message, source, triggerMessageIds, addressSignal, generation, usedWebSearchFollowup, reason: "invalid_structured_output_after_tool_loop", reaction: null, screenShareOffer: null, performance, prompts: replyPrompts, extraMetadata: { rawTextPreview: sanitizeBotText(String(generation.text || ""), 280) || null } }); return { skipped: true }; } const recoveredText = sanitizeBotText(String(generation.text || "")); if (recoveredText) { replyDirective.text = recoveredText; bot.store.logAction({ kind: "bot_warning", guildId: message.guildId, channelId: message.channelId, messageId: message.id, userId: bot.client.user?.id || null, content: "structured_output_recovered_as_prose", metadata: { source, parseState: replyDirective.parseState } }); } else { bot.logSkippedReply({ message, source, triggerMessageIds, addressSignal, generation, usedWebSearchFollowup, reason: "invalid_structured_output", reaction: null, screenShareOffer: null, performance, prompts: replyPrompts, extraMetadata: { rawTextPreview: sanitizeBotText(String(generation.text || ""), 280) || null } }); return { skipped: true }; } }

const reaction = await bot.maybeApplyReplyReaction({ message, settings, emojiOptions: reactionEmojiOptions, emojiToken: replyDirective.reactionEmoji, generation, source, triggerMessageId: message.id, triggerMessageIds, addressing: addressSignal });

const mediaDirective = pickReplyMediaDirective(replyDirective); let finalText = sanitizeBotText(replyDirective.text || ""); let mentionResolution = emptyMentionResolution(); finalText = normalizeSkipSentinel(finalText); const screenShareOffer = await bot.maybeHandleScreenWatchIntent({ message, replyDirective, source }); if (screenShareOffer.appendText) { const textParts = []; if (finalText && finalText !== "[SKIP]") textParts.push(finalText); textParts.push(screenShareOffer.appendText); finalText = sanitizeBotText(textParts.join(" "), 1700); } const allowMediaOnlyReply = !finalText && Boolean(mediaDirective); const modelProducedSkip = finalText === "[SKIP]"; const modelProducedEmpty = !finalText; if (modelProducedEmpty && !allowMediaOnlyReply) { bot.store.logAction({ kind: "bot_error", guildId: message.guildId, channelId: message.channelId, messageId: message.id, userId: bot.client.user?.id || null, content: "reply_model_output_empty", metadata: { source, triggerMessageIds, addressed: Boolean(addressSignal?.triggered) } }); } if (finalText === "[SKIP]" || (!finalText && !allowMediaOnlyReply)) { bot.logSkippedReply({ message, source, triggerMessageIds, addressSignal, generation, usedWebSearchFollowup, reason: modelProducedSkip ? "llm_skip" : "empty_reply", reaction, screenShareOffer, performance, prompts: replyPrompts }); return { skipped: true }; }

mentionResolution = await resolveDeterministicMentions( { store: bot.store }, { text: finalText, guild: message.guild, guildId: message.guildId } ); finalText = mentionResolution.text; finalText = embedWebSearchSources(finalText, webSearch);

let payload: ReplyMessagePayload = { content: finalText }; let toolImagesUsed = false; let imageUsed = false; let imageBudgetBlocked = false; let imageCapabilityBlocked = false; let imageVariantUsed = null; let videoUsed = false; let videoBudgetBlocked = false; let videoCapabilityBlocked = false; let gifUsed = false; let gifBudgetBlocked = false; let gifConfigBlocked = false; const imagePrompt = replyDirective.imagePrompt; const complexImagePrompt = replyDirective.complexImagePrompt; const videoPrompt = replyDirective.videoPrompt; const gifQuery = replyDirective.gifQuery; const mediaAttachment = await bot.resolveMediaAttachment({ settings, text: finalText, directive: { type: mediaDirective?.type ?? null, gifQuery, imagePrompt: mediaDirective?.type === "image_simple" && discovery.allowReplyImages && imagePrompt ? composeReplyImagePrompt( imagePrompt, finalText, mediaPromptLimit, replyMediaMemoryFacts ) : null, complexImagePrompt: mediaDirective?.type === "image_complex" && discovery.allowReplyImages && complexImagePrompt ? composeReplyImagePrompt( complexImagePrompt, finalText, mediaPromptLimit, replyMediaMemoryFacts ) : null, videoPrompt: mediaDirective?.type === "video" && discovery.allowReplyVideos && videoPrompt ? composeReplyVideoPrompt( videoPrompt, finalText, mediaPromptLimit, replyMediaMemoryFacts ) : null }, toolImageInputs: toolResultImageInputs, trace: { guildId: message.guildId, channelId: message.channelId, userId: message.author.id, source: "reply_message" } }); payload = mediaAttachment.payload; toolImagesUsed = mediaAttachment.toolImagesUsed; imageUsed = mediaAttachment.imageUsed; imageBudgetBlocked = mediaAttachment.imageBudgetBlocked; imageCapabilityBlocked = mediaAttachment.imageCapabilityBlocked; imageVariantUsed = mediaAttachment.imageVariantUsed; videoUsed = mediaAttachment.videoUsed; videoBudgetBlocked = mediaAttachment.videoBudgetBlocked; videoCapabilityBlocked = mediaAttachment.videoCapabilityBlocked; gifUsed = mediaAttachment.gifUsed; gifBudgetBlocked = mediaAttachment.gifBudgetBlocked; gifConfigBlocked = mediaAttachment.gifConfigBlocked;

if (!finalText && !toolImagesUsed && !imageUsed && !videoUsed && !gifUsed) { bot.store.logAction({ kind: "bot_error", guildId: message.guildId, channelId: message.channelId, messageId: message.id, userId: bot.client.user?.id || null, content: "reply_model_output_empty_after_media", metadata: { source, triggerMessageIds, addressed: Boolean(addressSignal?.triggered) } }); bot.logSkippedReply({ message, source, triggerMessageIds, addressSignal, generation, usedWebSearchFollowup, reason: "empty_reply_after_media", reaction, screenShareOffer, performance, prompts: replyPrompts }); return { skipped: true }; }

return { skipped: false, reaction, mediaDirective, finalText, mentionResolution, screenShareOffer, allowMediaOnlyReply, modelProducedSkip, modelProducedEmpty, payload, toolImagesUsed, imageUsed, imageBudgetBlocked, imageCapabilityBlocked, imageVariantUsed, videoUsed, videoBudgetBlocked, videoCapabilityBlocked, gifUsed, gifBudgetBlocked, gifConfigBlocked, imagePrompt, complexImagePrompt, videoPrompt, gifQuery }; }

async function sendReplyMessage( bot: ReplyPipelineRuntime, message: ReplyPipelineMessage, settings: Settings, options: ReplyAttemptOptions, ctx: ReplyPipelineContext, llmResult: ReplyActionableLlmResult, actionResult: ReplySendableActionResult ): Promise { const botName = getBotName(settings); const { addressSignal, triggerMessageIds, addressed, isReplyChannel, source, performance, imageBudget, videoBudget, simpleImageCapabilityReady, complexImageCapabilityReady, imageCapabilityReady, videoCapabilityReady, gifBudget } = ctx; const { generation, typingDelayMs, usedWebSearchFollowup, usedMemoryLookupFollowup, usedImageLookupFollowup, toolResultImageInputs, webSearch, imageLookup, memoryLookup, replyPrompts } = llmResult; const { reaction, mediaDirective, finalText, mentionResolution, screenShareOffer, payload, toolImagesUsed, imageUsed, imageBudgetBlocked, imageCapabilityBlocked, imageVariantUsed, videoUsed, videoBudgetBlocked, videoCapabilityBlocked, gifUsed, gifBudgetBlocked, gifConfigBlocked, imagePrompt, complexImagePrompt, videoPrompt, gifQuery } = actionResult;

const shouldThreadReply = addressed; const isDiscovery = bot.isDiscoveryChannel(settings, message.channelId); const canStandalonePost = isReplyChannel || isDiscovery || !shouldThreadReply; const sendAsReply = bot.shouldSendAsReply({ isReplyChannel, shouldThreadReply, replyText: finalText }); const sendStartedAtMs = Date.now(); const textChunks = splitDiscordMessage(payload.content); const firstPayload = { ...payload, content: textChunks[0] }; const sent = sendAsReply ? await message.reply({ ...firstPayload, allowedMentions: { repliedUser: false } }) : await message.channel.send(firstPayload); for (let i = 1; i < textChunks.length; i++) { await message.channel.send({ content: textChunks[i] }); } const sendMs = Math.max(0, Date.now() - sendStartedAtMs); const actionKind = sendAsReply ? "sent_reply" : "sent_message"; const referencedMessageId = sendAsReply ? message.id : null; const memorySettings = getMemorySettings(settings);

bot.markSpoke(); bot.store.recordMessage({ messageId: sent.id, createdAt: sent.createdTimestamp, guildId: sent.guildId, channelId: sent.channelId, authorId: bot.client.user.id, authorName: botName, isBot: true, content: bot.composeMessageContentForHistory(sent, finalText), referencedMessageId }); if (memorySettings.enabled && typeof bot.memory?.ingestMessage === "function") { void bot.memory.ingestMessage({ messageId: sent.id, authorId: bot.client.user.id, authorName: botName, content: finalText, isBot: true, settings, trace: { guildId: sent.guildId, channelId: sent.channelId, userId: bot.client.user.id, source: "text_reply_memory_ingest" } }).catch((error) => { bot.store.logAction({ kind: "bot_error", guildId: sent.guildId, channelId: sent.channelId, messageId: sent.id, userId: bot.client.user.id, content: memory_text_reply_ingest: ${String(error?.message || error)} }); }); } bot.store.logAction({ kind: actionKind, guildId: sent.guildId, channelId: sent.channelId, messageId: sent.id, userId: bot.client.user.id, content: finalText, metadata: { triggerMessageId: message.id, triggerMessageIds, source, addressing: addressSignal, replyPrompts, sendAsReply, canStandalonePost, image: { requestedByModel: Boolean(imagePrompt || complexImagePrompt), requestedSimpleByModel: Boolean(imagePrompt), requestedComplexByModel: Boolean(complexImagePrompt), selectedVariant: imageVariantUsed, used: imageUsed, blockedByDailyCap: imageBudgetBlocked, blockedByCapability: imageCapabilityBlocked, maxPerDay: imageBudget.maxPerDay, remainingAtPromptTime: imageBudget.remaining, simpleCapabilityReadyAtPromptTime: simpleImageCapabilityReady, complexCapabilityReadyAtPromptTime: complexImageCapabilityReady, capabilityReadyAtPromptTime: imageCapabilityReady }, videoGeneration: { requestedByModel: Boolean(videoPrompt), used: videoUsed, blockedByDailyCap: videoBudgetBlocked, blockedByCapability: videoCapabilityBlocked, maxPerDay: videoBudget.maxPerDay, remainingAtPromptTime: videoBudget.remaining, capabilityReadyAtPromptTime: videoCapabilityReady }, gif: { requestedByModel: Boolean(gifQuery), used: gifUsed, blockedByDailyCap: gifBudgetBlocked, blockedByConfiguration: gifConfigBlocked, maxPerDay: gifBudget.maxPerDay, remainingAtPromptTime: gifBudget.remaining }, toolImages: { requestedByModel: mediaDirective?.type === "tool_images", availableFromTools: toolResultImageInputs.length, used: toolImagesUsed }, memory: { toolCallsUsed: usedMemoryLookupFollowup, query: memoryLookup?.query || null, results: (memoryLookup?.results || []).map((r: Record<string, unknown>) => ({ fact: r.fact, fact_type: r.fact_type, subject: r.subject, confidence: r.confidence })) }, imageLookup: { requested: imageLookup.requested, used: imageLookup.used, query: imageLookup.query, candidateCount: imageLookup.candidates?.length || 0, resultCount: imageLookup.results?.length || 0, error: imageLookup.error || null, results: (imageLookup.results || []).map((r: Record<string, unknown>) => ({ filename: r.filename, authorName: r.authorName, url: r.url, matchReason: r.matchReason })) }, mentions: mentionResolution, reaction, screenShareOffer, webSearch: { requested: webSearch.requested, used: webSearch.used, query: webSearch.query, resultCount: webSearch.results?.length || 0, results: (webSearch.results || []).map((r) => ({ title: r.title, url: r.url, domain: r.domain })), fetchedPages: webSearch.fetchedPages || 0, providerUsed: webSearch.providerUsed || null, providerFallbackUsed: Boolean(webSearch.providerFallbackUsed), blockedByHourlyCap: webSearch.blockedByBudget, maxPerHour: webSearch.budget?.maxPerHour ?? null, remainingAtPromptTime: webSearch.budget?.remaining ?? null, configured: webSearch.configured, optedOutByUser: webSearch.optedOutByUser, error: webSearch.error || null }, video: { mode: "agent_tool" }, llm: { provider: generation.provider, model: generation.model, usage: generation.usage, costUsd: generation.costUsd, usedWebSearchFollowup, usedMemoryLookupFollowup, usedImageLookupFollowup }, performance: finalizeReplyPerformanceSample({ performance, actionKind, typingDelayMs, sendMs }) } });

return true; }

export async function maybeReplyToMessagePipeline( bot: ReplyPipelineRuntime, message: ReplyPipelineMessage, settings: Settings, options: ReplyAttemptOptions = {} ): Promise { const permissions = getReplyPermissions(settings); if (!permissions.allowReplies) { logReplyPipelineGate(bot, { message, settings, options, allow: false, reason: "replies_disabled", sendBudgetAllowed: null, talkNowAllowed: null }); return false; } const sendBudgetAllowed = bot.canSendMessage(permissions.maxMessagesPerHour); if (!sendBudgetAllowed) { logReplyPipelineGate(bot, { message, settings, options, allow: false, reason: "send_budget_blocked", sendBudgetAllowed, talkNowAllowed: null }); return false; } const talkNowAllowed = bot.canTalkNow(settings); if (!talkNowAllowed) { logReplyPipelineGate(bot, { message, settings, options, allow: false, reason: "talk_now_blocked", sendBudgetAllowed, talkNowAllowed }); return false; }

const replyScopeKey = buildTextReplyScopeKey({ guildId: message.guildId, channelId: message.channelId }); const activeReply = bot.activeReplies.begin(replyScopeKey, "text-reply"); const signal = activeReply.abortController.signal;

try { throwIfAborted(signal, "Reply cancelled"); const ctx = await buildReplyContext(bot, message, settings, options); if (!ctx || !ctx.shouldRun) { logReplyPipelineGate(bot, { message, settings, options, allow: false, reason: "context_unavailable", sendBudgetAllowed, talkNowAllowed, ctx: ctx || null }); return false; } ctx.signal = signal; logReplyPipelineGate(bot, { message, settings, options, allow: true, reason: "ready", sendBudgetAllowed, talkNowAllowed, ctx });

throwIfAborted(signal, "Reply cancelled");
const llmResult = await executeReplyLlm(bot, message, settings, options, ctx);
if (!isReplyActionableLlmResult(llmResult)) return true;
const actionableLlmResult = llmResult;

throwIfAborted(signal, "Reply cancelled");
const actionResult = await dispatchReplyActions(bot, message, settings, options, ctx, actionableLlmResult);
if (!isReplySendableActionResult(actionResult)) return false;
const sendableActionResult = actionResult;

throwIfAborted(signal, "Reply cancelled");
return await sendReplyMessage(
  bot,
  message,
  settings,
  options,
  ctx,
  actionableLlmResult,
  sendableActionResult
);

} catch (error) { if (isAbortError(error) || signal.aborted) { // Return true ("reply handled") so the caller does not retry or fall back // to another reply path after the user explicitly cancelled this turn. return true; } throw error; } finally { bot.activeReplies.clear(activeReply); } }