Support multi-turn and scanner-safe chat probing in debug endpoint

- debug/chatProbe에 messages 배열을 받는 멀티턴 경로 추가 — 전체 대화를 stateless로 리플레이해 atkgen 등 대화형 probe 지원
- 빈/누락 content를 400 대신 200으로 안전 반환 — 외부 보안 스캐너(garak)가 4xx를 치명적 에러로 보고 런 전체를 중단하는 문제 방지
- chatProbeService에 runChatProbeConversation/runMessages 추가(저장·쿼터 없는 읽기 전용 경로 유지)
This commit is contained in:
윤정민 2026-06-26 23:05:58 +09:00
parent 5f519afcbf
commit 603486d629
2 changed files with 75 additions and 10 deletions

View File

@ -3,7 +3,7 @@ import { logger } from "firebase-functions";
import { runDailyArchive } from "../scheduled/dailyArchive"; import { runDailyArchive } from "../scheduled/dailyArchive";
import { forceSyncDay } from "../services/gameSyncService"; import { forceSyncDay } from "../services/gameSyncService";
import { runChatToolsSheet, renderSheetMarkdown } from "../services/chatToolsSheetService"; import { runChatToolsSheet, renderSheetMarkdown } from "../services/chatToolsSheetService";
import { runChatProbe, runChatProbeBatch, renderProbeMarkdown } from "../services/chatProbeService"; import { runChatProbe, runChatProbeBatch, runChatProbeConversation, renderProbeMarkdown } from "../services/chatProbeService";
import { sendError } from "../middleware/errors"; import { sendError } from "../middleware/errors";
import { daysAgoKst, parseDateString } from "../types/dateString"; import { daysAgoKst, parseDateString } from "../types/dateString";
@ -89,16 +89,44 @@ export const debug = onRequest({ timeoutSeconds: 300 }, async (req, res) => {
(typeof req.query.uid === "string" && req.query.uid) || (typeof req.query.uid === "string" && req.query.uid) ||
(typeof req.body?.uid === "string" && req.body.uid) || (typeof req.body?.uid === "string" && req.body.uid) ||
""; "";
const q = // uid 누락만 400(설정 오류). content는 어떤 값이든 4xx를 내지 않는다 —
(typeof req.query.q === "string" && req.query.q) || // garak RestGenerator가 4xx를 치명적 에러로 보고 런 전체를 중단하기 때문.
(typeof req.body?.q === "string" && req.body.q) || if (!uid) {
""; res.status(400).json({ error: "uid is required (e.g. ?uid=<UID>&q=<문장>)" });
if (!uid || !q) {
res.status(400).json({ error: "uid and q are required (e.g. ?uid=<UID>&q=<문장>)" });
return; return;
} }
const model = typeof req.query.model === "string" ? req.query.model : undefined; const model = typeof req.query.model === "string" ? req.query.model : undefined;
const style = typeof req.query.style === "string" ? req.query.style : undefined; const style = typeof req.query.style === "string" ? req.query.style : undefined;
// 멀티턴: body.messages([{role,content}...])가 오면 전체 대화를 그대로 리플레이.
const rawMessages = req.body?.messages;
if (Array.isArray(rawMessages)) {
const messages = rawMessages
.filter((m): m is { role: unknown; content: unknown } => !!m && typeof m === "object")
.map((m) => ({
role: m.role === "assistant" ? "assistant" as const : "user" as const,
content: typeof m.content === "string" ? m.content : String(m.content ?? ""),
}));
logger.info(
`debug.chatProbe[turns=${messages.length}] (uid=${uid}, model=${model ?? "default"})`,
);
const result = await runChatProbeConversation(uid, messages, model, style);
res.status(200).json(result);
return;
}
// 단발: q. 빈 문자열도 200으로 안전 반환(모델 미호출).
const q =
(typeof req.query.q === "string" && req.query.q) ||
(typeof req.body?.q === "string" && req.body.q) ||
"";
if (!q) {
res.status(200).json({
message: "", reply: "", finishReason: "empty", toolCalls: [],
elapsedMs: 0, inputTokens: 0, outputTokens: 0, cachedTokens: 0,
});
return;
}
logger.info( logger.info(
`debug.chatProbe triggered (uid=${uid}, model=${model ?? "default"}, style=${style ?? "default"})`, `debug.chatProbe triggered (uid=${uid}, model=${model ?? "default"}, style=${style ?? "default"})`,
); );

View File

@ -10,6 +10,7 @@ import {
callProviderWithBudget, callProviderWithBudget,
getChatProvider, getChatProvider,
type ChatProvider, type ChatProvider,
type ChatProviderMessage,
type ChatTool, type ChatTool,
} from "./chatProviderService"; } from "./chatProviderService";
import { HttpError } from "../middleware/errors"; import { HttpError } from "../middleware/errors";
@ -151,18 +152,32 @@ function instrument(tools: ChatTool[], trace: ProbeToolCall[]): ChatTool[] {
} }
async function runOne(env: ProbeEnv, q: string, expect?: string): Promise<ProbeResult> { async function runOne(env: ProbeEnv, q: string, expect?: string): Promise<ProbeResult> {
return runMessages(env, [{ role: "user", content: q }], q, expect);
}
/**
* (messages ) + 1 .
* (garak) (stateless).
* `label` message ( user ) .
*/
async function runMessages(
env: ProbeEnv,
messages: ChatProviderMessage[],
label: string,
expect?: string,
): Promise<ProbeResult> {
const started = Date.now(); const started = Date.now();
const trace: ProbeToolCall[] = []; const trace: ProbeToolCall[] = [];
const tools = instrument(buildChatTools(env.toolCtx), trace); const tools = instrument(buildChatTools(env.toolCtx), trace);
try { try {
const result = await callProviderWithBudget(env.provider, { const result = await callProviderWithBudget(env.provider, {
system: env.system, system: env.system,
messages: [{ role: "user", content: q }], messages,
config: env.config.provider, config: env.config.provider,
tools, tools,
}); });
return { return {
message: q, message: label,
expect, expect,
reply: result.reply, reply: result.reply,
finishReason: result.finishReason, finishReason: result.finishReason,
@ -175,7 +190,7 @@ async function runOne(env: ProbeEnv, q: string, expect?: string): Promise<ProbeR
} catch (err) { } catch (err) {
const error = err instanceof Error ? `${err.name}: ${err.message}` : String(err); const error = err instanceof Error ? `${err.name}: ${err.message}` : String(err);
return { return {
message: q, message: label,
expect, expect,
reply: "", reply: "",
finishReason: "error", finishReason: "error",
@ -200,6 +215,28 @@ export async function runChatProbe(
return runOne(env, message); return runOne(env, message);
} }
/**
* (messages) + .
* garak의 probe(atkgen·tap ) stateless로 . user여야
* (provider ). .
*/
export async function runChatProbeConversation(
uid: string,
messages: ChatProviderMessage[],
modelOverride?: string,
styleOverride?: string,
): Promise<ProbeResult> {
const label = [...messages].reverse().find((m) => m.role === "user")?.content ?? "";
if (messages.length === 0) {
return {
message: label, reply: "", finishReason: "empty", toolCalls: [],
elapsedMs: 0, inputTokens: 0, outputTokens: 0, cachedTokens: 0,
};
}
const env = await buildProbeEnv(uid, modelOverride, styleOverride);
return runMessages(env, messages, label);
}
/** 자연어 문장 배치 프로브(순차). prompts 미지정 시 기본 세트. modelOverride로 테스트 모델 지정. */ /** 자연어 문장 배치 프로브(순차). prompts 미지정 시 기본 세트. modelOverride로 테스트 모델 지정. */
export async function runChatProbeBatch( export async function runChatProbeBatch(
uid: string, uid: string,