diff --git a/backend/internal/service/content_moderation.go b/backend/internal/service/content_moderation.go index b25f5ec4dc..66d38b52b8 100644 --- a/backend/internal/service/content_moderation.go +++ b/backend/internal/service/content_moderation.go @@ -909,30 +909,10 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer "configured_models", cfg.ModelFilter.Models) return allow, nil } - content := ExtractContentModerationInput(input.Protocol, input.Body) - if content.IsEmpty() { - slog.Info("content_moderation.skip_empty_input", - "user_id", input.UserID, - "api_key_id", input.APIKeyID, - "group_id", contentModerationLogGroupID(input.GroupID), - "endpoint", input.Endpoint, - "protocol", input.Protocol, - "body_bytes", len(input.Body)) - return allow, nil - } - content.Normalize() - slog.Info("content_moderation.input_extracted", - "user_id", input.UserID, - "api_key_id", input.APIKeyID, - "group_id", contentModerationLogGroupID(input.GroupID), - "endpoint", input.Endpoint, - "protocol", input.Protocol, - "text_runes", len([]rune(content.Text)), - "image_count", len(content.Images)) - hashText := content.Hash() if cfg.Mode == ContentModerationModePreBlock { if cfg.KeywordBlockingMode != ContentModerationKeywordModeAPIOnly && len(cfg.BlockedKeywords) > 0 { - if keyword, hit := runtimeSnapshot.matchBlockedKeyword(content.Text); hit { + keywordText := extractContentModerationKeywordText(input.Protocol, input.Body) + if keyword, hit := runtimeSnapshot.matchBlockedKeyword(keywordText); hit { s.recordPreBlockSyncMetric(0, ContentModerationActionKeywordBlock) slog.Info("content_moderation.keyword_block", "user_id", input.UserID, @@ -943,9 +923,9 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer "keyword_blocking_mode", cfg.KeywordBlockingMode, "keyword", keyword) scores := map[string]float64{contentModerationKeywordCategory: 1.0} - log := s.buildLog(input, cfg, ContentModerationActionKeywordBlock, true, contentModerationKeywordCategory, 1.0, scores, content.ExcerptText(), nil, nil, "") + log := s.buildLog(input, cfg, ContentModerationActionKeywordBlock, true, contentModerationKeywordCategory, 1.0, scores, keywordText, nil, nil, "") log.MatchedKeyword = keyword - s.enqueueRecord(input, cfg, log, hashText, false, true) + s.enqueueRecord(input, cfg, log, (ContentModerationInput{Text: keywordText}).Hash(), false, true) return &ContentModerationDecision{ Allowed: false, Blocked: true, @@ -970,6 +950,27 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer return allow, nil } } + content := ExtractContentModerationInput(input.Protocol, input.Body) + if content.IsEmpty() { + slog.Info("content_moderation.skip_empty_input", + "user_id", input.UserID, + "api_key_id", input.APIKeyID, + "group_id", contentModerationLogGroupID(input.GroupID), + "endpoint", input.Endpoint, + "protocol", input.Protocol, + "body_bytes", len(input.Body)) + return allow, nil + } + content.Normalize() + slog.Info("content_moderation.input_extracted", + "user_id", input.UserID, + "api_key_id", input.APIKeyID, + "group_id", contentModerationLogGroupID(input.GroupID), + "endpoint", input.Endpoint, + "protocol", input.Protocol, + "text_runes", len([]rune(content.Text)), + "image_count", len(content.Images)) + hashText := content.Hash() if cfg.PreHashCheckEnabled && s.hashCache != nil { matched, err := s.hashCache.HasFlaggedInputHash(ctx, hashText) if err != nil { diff --git a/backend/internal/service/content_moderation_input.go b/backend/internal/service/content_moderation_input.go index 73d5b6973a..013c10c13d 100644 --- a/backend/internal/service/content_moderation_input.go +++ b/backend/internal/service/content_moderation_input.go @@ -14,6 +14,21 @@ func ExtractContentModerationText(protocol string, body []byte) string { } func ExtractContentModerationInput(protocol string, body []byte) ContentModerationInput { + return extractContentModerationInput(protocol, body, true) +} + +// Keyword checks share semantic moderation's current-user boundaries, but must +// inspect client-supplied reminder blocks as ordinary user text. +func extractContentModerationKeywordText(protocol string, body []byte) string { + return extractContentModerationInput(protocol, body, false).Text +} + +type moderationTextCollector struct { + filterReminders bool +} + +func extractContentModerationInput(protocol string, body []byte, filterReminders bool) ContentModerationInput { + collector := moderationTextCollector{filterReminders: filterReminders} if len(body) == 0 || !gjson.ValidBytes(body) { return ContentModerationInput{} } @@ -21,30 +36,32 @@ func ExtractContentModerationInput(protocol string, body []byte) ContentModerati var images []string switch protocol { case ContentModerationProtocolAnthropicMessages: - collectLastAnthropicUserMessage(gjson.GetBytes(body, "messages"), &parts, &images) + collector.collectLastAnthropicUserMessage(gjson.GetBytes(body, "messages"), &parts, &images) case ContentModerationProtocolOpenAIChat: - collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images) + collector.collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images) case ContentModerationProtocolOpenAIResponses: - collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images) + collector.collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images) case ContentModerationProtocolGemini: - collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images) + collector.collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images) case ContentModerationProtocolOpenAIImages: - addModerationText(&parts, gjson.GetBytes(body, "prompt").String()) - collectContentValue(gjson.GetBytes(body, "images"), &parts, &images) + collector.addModerationText(&parts, gjson.GetBytes(body, "prompt").String()) + collector.collectContentValue(gjson.GetBytes(body, "images"), &parts, &images) default: - collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images) - collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images) - collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images) + collector.collectLastResponsesInput(gjson.GetBytes(body, "input"), &parts, &images) + collector.collectLastRoleMessage(gjson.GetBytes(body, "messages"), "user", &parts, &images) + collector.collectLastGeminiContent(gjson.GetBytes(body, "contents"), &parts, &images) } out := ContentModerationInput{ Text: normalizeContentModerationText(strings.Join(parts, "\n")), Images: normalizeModerationImages(images), } - out.Normalize() + if filterReminders { + out.Normalize() + } return out } -func collectLastRoleMessage(messages gjson.Result, role string, parts *[]string, images *[]string) { +func (collector moderationTextCollector) collectLastRoleMessage(messages gjson.Result, role string, parts *[]string, images *[]string) { if !messages.IsArray() { return } @@ -58,7 +75,7 @@ func collectLastRoleMessage(messages gjson.Result, role string, parts *[]string, } var candidate []string var candidateImages []string - collectContentValue(last.Get("content"), &candidate, &candidateImages) + collector.collectContentValue(last.Get("content"), &candidate, &candidateImages) if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 { return } @@ -66,7 +83,7 @@ func collectLastRoleMessage(messages gjson.Result, role string, parts *[]string, *images = append(*images, candidateImages...) } -func collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, images *[]string) { +func (collector moderationTextCollector) collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, images *[]string) { if !messages.IsArray() { return } @@ -80,7 +97,7 @@ func collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, ima } var candidate []string var candidateImages []string - collectAnthropicUserContentValue(last.Get("content"), &candidate, &candidateImages) + collector.collectAnthropicUserContentValue(last.Get("content"), &candidate, &candidateImages) if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 { return } @@ -88,31 +105,31 @@ func collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, ima *images = append(*images, candidateImages...) } -func collectAnthropicUserContentValue(value gjson.Result, parts *[]string, images *[]string) { +func (collector moderationTextCollector) collectAnthropicUserContentValue(value gjson.Result, parts *[]string, images *[]string) { switch { case !value.Exists(): return case value.Type == gjson.String: - if !isAnthropicSystemReminderText(value.String()) { - addModerationText(parts, value.String()) + if !collector.filterReminders || !isAnthropicSystemReminderText(value.String()) { + collector.addModerationText(parts, value.String()) } case value.IsArray(): value.ForEach(func(_, item gjson.Result) bool { - collectAnthropicUserContentValue(item, parts, images) + collector.collectAnthropicUserContentValue(item, parts, images) return true }) case value.IsObject(): typ := strings.ToLower(strings.TrimSpace(value.Get("type").String())) switch typ { case "", "text", "input_text", "message": - if value.Get("text").Exists() && !isAnthropicSystemReminderText(value.Get("text").String()) { - addModerationText(parts, value.Get("text").String()) + if value.Get("text").Exists() && (!collector.filterReminders || !isAnthropicSystemReminderText(value.Get("text").String())) { + collector.addModerationText(parts, value.Get("text").String()) } if value.Get("content").Exists() { - collectAnthropicUserContentValue(value.Get("content"), parts, images) + collector.collectAnthropicUserContentValue(value.Get("content"), parts, images) } case "image_url", "input_image", "image": - collectContentValue(value, parts, images) + collector.collectContentValue(value, parts, images) } } } @@ -121,57 +138,57 @@ func isAnthropicSystemReminderText(text string) bool { return strings.HasPrefix(strings.TrimSpace(text), "") } -func collectLastResponsesInput(input gjson.Result, parts *[]string, images *[]string) { +func (collector moderationTextCollector) collectLastResponsesInput(input gjson.Result, parts *[]string, images *[]string) { switch { case !input.Exists(): return case input.Type == gjson.String: - addModerationText(parts, input.String()) + collector.addModerationText(parts, input.String()) case input.IsArray(): array := input.Array() if len(array) == 0 { return } last := array[len(array)-1] - if !isResponsesUserTextItem(last) { + if !collector.isResponsesUserTextItem(last) { return } - collectContentValue(last.Get("content"), parts, images) + collector.collectContentValue(last.Get("content"), parts, images) if last.Get("type").String() == "input_text" || last.Get("text").Exists() { - collectContentValue(last, parts, images) + collector.collectContentValue(last, parts, images) } case input.IsObject(): - if isResponsesUserTextItem(input) { - collectContentValue(input.Get("content"), parts, images) + if collector.isResponsesUserTextItem(input) { + collector.collectContentValue(input.Get("content"), parts, images) if input.Get("type").String() == "input_text" || input.Get("text").Exists() { - collectContentValue(input, parts, images) + collector.collectContentValue(input, parts, images) } } } } -func isResponsesUserTextItem(item gjson.Result) bool { +func (collector moderationTextCollector) isResponsesUserTextItem(item gjson.Result) bool { role := strings.ToLower(strings.TrimSpace(item.Get("role").String())) if role == "user" { - return responseItemHasModerationText(item) + return collector.responseItemHasModerationText(item) } if role != "" { return false } - return responseItemHasModerationText(item) + return collector.responseItemHasModerationText(item) } -func responseItemHasModerationText(item gjson.Result) bool { +func (collector moderationTextCollector) responseItemHasModerationText(item gjson.Result) bool { var parts []string var images []string - collectContentValue(item.Get("content"), &parts, &images) + collector.collectContentValue(item.Get("content"), &parts, &images) if item.Get("type").String() == "input_text" || item.Get("text").Exists() { - collectContentValue(item, &parts, &images) + collector.collectContentValue(item, &parts, &images) } return normalizeContentModerationText(strings.Join(parts, "\n")) != "" || len(images) > 0 } -func collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]string) { +func (collector moderationTextCollector) collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]string) { if !contents.IsArray() { return } @@ -188,7 +205,7 @@ func collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[] var candidateImages []string if arr := last.Get("parts"); arr.IsArray() { arr.ForEach(func(_, part gjson.Result) bool { - addModerationText(&candidate, part.Get("text").String()) + collector.addModerationText(&candidate, part.Get("text").String()) addGeminiModerationImage(&candidateImages, part) return true }) @@ -200,15 +217,15 @@ func collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[] *images = append(*images, candidateImages...) } -func collectContentValue(value gjson.Result, parts *[]string, images *[]string) { +func (collector moderationTextCollector) collectContentValue(value gjson.Result, parts *[]string, images *[]string) { switch { case !value.Exists(): return case value.Type == gjson.String: - addModerationText(parts, value.String()) + collector.addModerationText(parts, value.String()) case value.IsArray(): value.ForEach(func(_, item gjson.Result) bool { - collectContentValue(item, parts, images) + collector.collectContentValue(item, parts, images) return true }) case value.IsObject(): @@ -227,10 +244,10 @@ func collectContentValue(value gjson.Result, parts *[]string, images *[]string) switch typ { case "", "text", "input_text", "message": if value.Get("text").Exists() { - addModerationText(parts, value.Get("text").String()) + collector.addModerationText(parts, value.Get("text").String()) } if value.Get("content").Exists() { - collectContentValue(value.Get("content"), parts, images) + collector.collectContentValue(value.Get("content"), parts, images) } case "image_url", "input_image", "image": } @@ -303,12 +320,12 @@ func limitContentModerationImages(images []string) []string { return []string{images[int(idx.Int64())]} } -func addModerationText(parts *[]string, text string) { +func (collector moderationTextCollector) addModerationText(parts *[]string, text string) { text = strings.TrimSpace(text) if text == "" { return } - if strings.Contains(text, "") { + if collector.filterReminders && strings.Contains(text, "") { return } *parts = append(*parts, text) diff --git a/backend/internal/service/content_moderation_reminder_test.go b/backend/internal/service/content_moderation_reminder_test.go new file mode 100644 index 0000000000..eb2c320493 --- /dev/null +++ b/backend/internal/service/content_moderation_reminder_test.go @@ -0,0 +1,226 @@ +package service + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func reminderTestBody(t *testing.T, protocol string, texts []string) []byte { + t.Helper() + blocks := make([]map[string]string, 0, len(texts)) + for _, text := range texts { + blocks = append(blocks, map[string]string{"type": "text", "text": text}) + } + message := map[string]any{"role": "user", "content": blocks} + var body any + switch protocol { + case ContentModerationProtocolGemini: + body = map[string]any{"contents": []any{map[string]any{"role": "user", "parts": blocks}}} + case ContentModerationProtocolOpenAIResponses, "unknown": + body = map[string]any{"input": []any{message}} + case ContentModerationProtocolOpenAIImages: + body = map[string]any{"prompt": strings.Join(texts, "\n")} + default: + body = map[string]any{"messages": []any{message}} + } + raw, err := json.Marshal(body) + require.NoError(t, err) + return raw +} + +func TestContentModerationCheck_ReminderKeywordModes(t *testing.T) { + for _, protocol := range []string{ContentModerationProtocolAnthropicMessages, ContentModerationProtocolOpenAIChat, ContentModerationProtocolOpenAIResponses, ContentModerationProtocolGemini, ContentModerationProtocolOpenAIImages, "unknown"} { + for name, texts := range map[string][]string{ + "plain": {"今晚打老虎"}, + "prefix": {"context 今晚打老虎"}, + "suffix": {"今晚打老虎 context"}, + "separate": {"context", "今晚打老虎"}, + "inside": {"今晚打老虎"}, + } { + for _, mode := range []string{ContentModerationKeywordModeKeywordOnly, ContentModerationKeywordModeAPIOnly, ContentModerationKeywordModeKeywordAndAPI} { + t.Run(protocol+"/"+name+"/"+mode, func(t *testing.T) { + body := reminderTestBody(t, protocol, texts) + semantic := ExtractContentModerationInput(protocol, body) + wantSemantic := "" + if name == "plain" || (name == "separate" && protocol != ContentModerationProtocolOpenAIImages) { + wantSemantic = "今晚打老虎" + } + require.Equal(t, wantSemantic, semantic.Text) + require.Contains(t, extractContentModerationKeywordText(protocol, body), "今晚打老虎") + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + var payload struct { + Input string `json:"input"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Error(err) + } + if payload.Input != semantic.Text { + t.Errorf("semantic input changed: %q != %q", payload.Input, semantic.Text) + } + _ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{}}}) + })) + defer server.Close() + cfg := defaultContentModerationConfig() + cfg.Enabled = true + cfg.Mode = ContentModerationModePreBlock + cfg.KeywordBlockingMode = mode + cfg.BlockedKeywords = []string{"今晚打老虎"} + cfg.BaseURL = server.URL + cfg.APIKeys = []string{"test"} + cfg.RecordNonHits = false + cfg.PreHashCheckEnabled = true + raw, err := json.Marshal(cfg) + require.NoError(t, err) + repo := &contentModerationTestRepo{} + cache := &contentModerationTestHashCache{} + if mode != ContentModerationKeywordModeAPIOnly { + cache.hashes = map[string]struct{}{semantic.Hash(): {}} + } + svc := NewContentModerationService(&contentModerationTestSettingRepo{values: map[string]string{SettingKeyRiskControlEnabled: "true", SettingKeyContentModerationConfig: string(raw)}}, repo, cache, nil, nil, nil, nil, nil) + decision, err := svc.Check(context.Background(), ContentModerationCheckInput{Protocol: protocol, Body: body}) + require.NoError(t, err) + if mode == ContentModerationKeywordModeAPIOnly { + require.True(t, decision.Allowed) + want := int32(0) + if !semantic.IsEmpty() { + want = 1 + } + require.Equal(t, want, calls.Load()) + require.Empty(t, repo.snapshotLogs()) + } else { + require.Equal(t, ContentModerationActionKeywordBlock, decision.Action) + require.True(t, decision.Blocked) + require.Zero(t, calls.Load()) + require.Empty(t, cache.snapshotChecked()) + logs := requireContentModerationLogCount(t, repo, 1) + require.Equal(t, "今晚打老虎", logs[0].MatchedKeyword) + require.Contains(t, logs[0].InputExcerpt, "今晚打老虎") + require.Equal(t, int64(1), svc.asyncEnqueued.Load()) + } + }) + } + } + } +} + +func TestExtractContentModerationKeywordText_Boundaries(t *testing.T) { + for _, tc := range []struct{ protocol, body string }{ + {ContentModerationProtocolAnthropicMessages, `{"messages":[{"role":"user","content":"old"},{"role":"user","content":[{"type":"tool_result","content":"今晚打老虎"}]}]}`}, + {ContentModerationProtocolAnthropicMessages, `{"messages":[{"role":"user","content":"old"},{"role":"assistant","content":"今晚打老虎"}]}`}, + {ContentModerationProtocolOpenAIChat, `{"messages":[{"role":"user","content":"old"},{"role":"tool","content":"今晚打老虎"}]}`}, + {ContentModerationProtocolOpenAIChat, `{"messages":[{"role":"user","content":"old"},{"role":"assistant","content":"今晚打老虎"}]}`}, + {ContentModerationProtocolOpenAIResponses, `{"input":[{"role":"user","content":"old"},{"type":"function_call_output","output":"今晚打老虎"}]}`}, + {ContentModerationProtocolOpenAIResponses, `{"input":[{"role":"user","content":"old"},{"role":"assistant","content":"今晚打老虎"}]}`}, + {ContentModerationProtocolGemini, `{"contents":[{"role":"user","parts":[{"text":"old"}]},{"role":"user","parts":[{"functionResponse":{"response":{"text":"今晚打老虎"}}}]}]}`}, + {ContentModerationProtocolGemini, `{"contents":[{"role":"user","parts":[{"text":"old"}]},{"role":"model","parts":[{"text":"今晚打老虎"}]}]}`}, + } { + t.Run(tc.protocol+tc.body, func(t *testing.T) { + require.Empty(t, extractContentModerationKeywordText(tc.protocol, []byte(tc.body))) + }) + } +} + +func TestContentModerationCheck_ReminderPolicyPreserved(t *testing.T) { + for _, name := range []string{"keyword_miss", "combined_miss", "observe", "off", "disabled", "group_scope", "model_scope", "no_api_key", "zero_sample"} { + t.Run(name, func(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + var payload struct { + Input string `json:"input"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Error(err) + } + if payload.Input != "safe text" { + t.Errorf("unexpected semantic input: %q", payload.Input) + } + _ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{}}}) + })) + defer server.Close() + cfg := defaultContentModerationConfig() + cfg.Enabled = true + cfg.Mode = ContentModerationModePreBlock + cfg.BlockedKeywords = []string{"今晚打老虎"} + cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordAndAPI + cfg.BaseURL = server.URL + cfg.APIKeys = []string{"test"} + cfg.RecordNonHits = false + cfg.PreHashCheckEnabled = false + switch name { + case "keyword_miss": + cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordOnly + cfg.BlockedKeywords = []string{"absent"} + case "combined_miss": + cfg.BlockedKeywords = []string{"absent"} + case "observe": + cfg.Mode = ContentModerationModeObserve + case "off": + cfg.Mode = ContentModerationModeOff + case "disabled": + cfg.Enabled = false + case "group_scope": + cfg.AllGroups = false + cfg.GroupIDs = []int64{42} + case "model_scope": + cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterInclude, Models: []string{"other-model"}} + case "no_api_key": + cfg.APIKeys = nil + case "zero_sample": + cfg.SampleRate = 0 + } + raw, err := json.Marshal(cfg) + require.NoError(t, err) + repo := &contentModerationTestRepo{} + svc := NewContentModerationService(&contentModerationTestSettingRepo{values: map[string]string{SettingKeyRiskControlEnabled: "true", SettingKeyContentModerationConfig: string(raw)}}, repo, nil, nil, nil, nil, nil, nil) + body := reminderTestBody(t, ContentModerationProtocolAnthropicMessages, []string{"今晚打老虎", "safe text"}) + decision, err := svc.Check(context.Background(), ContentModerationCheckInput{Protocol: ContentModerationProtocolAnthropicMessages, Body: body}) + require.NoError(t, err) + if name == "no_api_key" || name == "zero_sample" { + require.True(t, decision.Blocked) + requireContentModerationLogCount(t, repo, 1) + } else { + require.True(t, decision.Allowed) + } + switch name { + case "observe": + require.Eventually(t, func() bool { return calls.Load() == 1 }, time.Second, time.Millisecond*10) + case "combined_miss": + require.Equal(t, int32(1), calls.Load()) + default: + require.Zero(t, calls.Load()) + } + }) + } +} + +func TestExtractContentModerationKeywordText_ResponsesInputForms(t *testing.T) { + for _, body := range []string{ + `{"input":"今晚打老虎"}`, + `{"input":{"role":"user","content":"今晚打老虎"}}`, + `{"input":[{"type":"input_text","text":"今晚打老虎"}]}`, + } { + require.Contains(t, extractContentModerationKeywordText(ContentModerationProtocolOpenAIResponses, []byte(body)), "今晚打老虎") + require.Empty(t, ExtractContentModerationText(ContentModerationProtocolOpenAIResponses, []byte(body))) + } +} + +func TestExtractContentModerationKeywordText_LongReminder(t *testing.T) { + // Reminder context must not consume the semantic API's text limit and hide + // the actual user keyword from the local check. + body := reminderTestBody(t, ContentModerationProtocolAnthropicMessages, []string{ + "" + strings.Repeat("x", maxModerationInputRunes) + "", "今晚打老虎", + }) + require.Contains(t, extractContentModerationKeywordText(ContentModerationProtocolAnthropicMessages, body), "今晚打老虎") + require.Equal(t, "今晚打老虎", ExtractContentModerationText(ContentModerationProtocolAnthropicMessages, body)) +}