mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-11-06 00:48:36 +00:00
UserRepository.Update and APIKeyRepository.Update rewrote the whole row on every call, regardless of which fields the caller meant to change. Several columns on those tables are maintained by dedicated atomic paths (balance deduction, quota and rate-limit counters, limit adjustments, activity timestamps), so a caller holding a slightly older snapshot could silently roll them back - a lost update. Both methods now take an explicit column mask and persist only the columns the caller declares; everything else keeps its current database value. - All user and API-key call sites declare exactly what they mutate, which turns admin edits and profile saves into genuine partial updates. - Email uniqueness locking/lookup and allowed_groups sync only run when those fields are part of the update. - UserUpdateFields deliberately has no balance/total_recharged members, so Update cannot touch them. New AdjustBalance/SetBalance apply the change in a single statement and return before/after values; admin balance adjustment uses them instead of read-modify-write. - promo_codes.used_count is no longer written by Update; it is only ever incremented by the redemption path. - The billing hot path that marks an API key quota-exhausted writes only status. - Dropped a no-op row write in RevokeAllUserTokens: users has no token_version column, so it persisted nothing while still overwriting concurrently-updated columns. Adds integration coverage that a stale snapshot cannot revert concurrent atomic writes, and unit coverage pinning the column set each entry point declares.
1860 lines
64 KiB
Go
1860 lines
64 KiB
Go
package service
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"log/slog"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/Wei-Shaw/sub2api/internal/pkg/pagination"
|
||
"github.com/stretchr/testify/require"
|
||
)
|
||
|
||
type contentModerationTestSettingRepo struct {
|
||
values map[string]string
|
||
}
|
||
|
||
func (r *contentModerationTestSettingRepo) Get(ctx context.Context, key string) (*Setting, error) {
|
||
if value, ok := r.values[key]; ok {
|
||
return &Setting{Key: key, Value: value}, nil
|
||
}
|
||
return nil, ErrSettingNotFound
|
||
}
|
||
|
||
func (r *contentModerationTestSettingRepo) GetValue(ctx context.Context, key string) (string, error) {
|
||
if value, ok := r.values[key]; ok {
|
||
return value, nil
|
||
}
|
||
return "", ErrSettingNotFound
|
||
}
|
||
|
||
func (r *contentModerationTestSettingRepo) Set(ctx context.Context, key, value string) error {
|
||
if r.values == nil {
|
||
r.values = map[string]string{}
|
||
}
|
||
r.values[key] = value
|
||
return nil
|
||
}
|
||
|
||
func (r *contentModerationTestSettingRepo) GetMultiple(ctx context.Context, keys []string) (map[string]string, error) {
|
||
out := map[string]string{}
|
||
for _, key := range keys {
|
||
if value, ok := r.values[key]; ok {
|
||
out[key] = value
|
||
}
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (r *contentModerationTestSettingRepo) SetMultiple(ctx context.Context, settings map[string]string) error {
|
||
if r.values == nil {
|
||
r.values = map[string]string{}
|
||
}
|
||
for key, value := range settings {
|
||
r.values[key] = value
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (r *contentModerationTestSettingRepo) GetAll(ctx context.Context) (map[string]string, error) {
|
||
out := make(map[string]string, len(r.values))
|
||
for key, value := range r.values {
|
||
out[key] = value
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (r *contentModerationTestSettingRepo) Delete(ctx context.Context, key string) error {
|
||
delete(r.values, key)
|
||
return nil
|
||
}
|
||
|
||
type contentModerationTestRepo struct {
|
||
mu sync.Mutex
|
||
logs []ContentModerationLog
|
||
}
|
||
|
||
func (r *contentModerationTestRepo) CreateLog(ctx context.Context, log *ContentModerationLog) error {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
if log != nil {
|
||
r.logs = append(r.logs, *log)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (r *contentModerationTestRepo) ListLogs(ctx context.Context, filter ContentModerationLogFilter) ([]ContentModerationLog, *pagination.PaginationResult, error) {
|
||
return nil, nil, nil
|
||
}
|
||
|
||
func (r *contentModerationTestRepo) CountFlaggedByUserSince(ctx context.Context, userID int64, since time.Time, excludeCyberPolicy bool) (int, error) {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
count := 0
|
||
for _, log := range r.logs {
|
||
if log.UserID == nil || *log.UserID != userID || !log.Flagged || log.Action == ContentModerationActionHashBlock {
|
||
continue
|
||
}
|
||
if excludeCyberPolicy && log.Action == ContentModerationActionCyberPolicy {
|
||
continue
|
||
}
|
||
if log.CreatedAt.IsZero() || log.CreatedAt.Before(since) {
|
||
continue
|
||
}
|
||
count++
|
||
}
|
||
return count, nil
|
||
}
|
||
|
||
func (r *contentModerationTestRepo) CleanupExpiredLogs(ctx context.Context, hitBefore time.Time, nonHitBefore time.Time) (*ContentModerationCleanupResult, error) {
|
||
return &ContentModerationCleanupResult{}, nil
|
||
}
|
||
|
||
func (r *contentModerationTestRepo) UpdateLogEmailSent(ctx context.Context, id int64, sent bool) error {
|
||
return nil
|
||
}
|
||
|
||
func (r *contentModerationTestRepo) snapshotLogs() []ContentModerationLog {
|
||
r.mu.Lock()
|
||
defer r.mu.Unlock()
|
||
out := make([]ContentModerationLog, len(r.logs))
|
||
copy(out, r.logs)
|
||
return out
|
||
}
|
||
|
||
func requireContentModerationLogCount(t *testing.T, repo *contentModerationTestRepo, want int) []ContentModerationLog {
|
||
t.Helper()
|
||
var logs []ContentModerationLog
|
||
require.Eventually(t, func() bool {
|
||
logs = repo.snapshotLogs()
|
||
return len(logs) == want
|
||
}, time.Second, 10*time.Millisecond)
|
||
return logs
|
||
}
|
||
|
||
func requireRecordedHashCount(t *testing.T, cache *contentModerationTestHashCache, want int) []string {
|
||
t.Helper()
|
||
var hashes []string
|
||
require.Eventually(t, func() bool {
|
||
hashes = cache.snapshotRecorded()
|
||
return len(hashes) == want
|
||
}, time.Second, 10*time.Millisecond)
|
||
return hashes
|
||
}
|
||
|
||
type contentModerationTestHashCache struct {
|
||
mu sync.Mutex
|
||
hashes map[string]struct{}
|
||
recorded []string
|
||
checked []string
|
||
deleted []string
|
||
hasResult bool
|
||
hasResultUsed bool
|
||
}
|
||
|
||
type contentModerationTestUserRepo struct {
|
||
user *User
|
||
updated []User
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) Create(ctx context.Context, user *User) error {
|
||
panic("unexpected Create call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) CreateWithEmailAliasGuard(ctx context.Context, user *User) error {
|
||
panic("unexpected CreateWithEmailAliasGuard call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) GetByID(ctx context.Context, id int64) (*User, error) {
|
||
if r.user == nil {
|
||
return nil, ErrUserNotFound
|
||
}
|
||
clone := *r.user
|
||
return &clone, nil
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) GetByEmail(ctx context.Context, email string) (*User, error) {
|
||
panic("unexpected GetByEmail call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) GetFirstAdmin(ctx context.Context) (*User, error) {
|
||
panic("unexpected GetFirstAdmin call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) Update(ctx context.Context, user *User, fields UserUpdateFields) error {
|
||
if user == nil {
|
||
return nil
|
||
}
|
||
clone := *user
|
||
r.updated = append(r.updated, clone)
|
||
r.user = &clone
|
||
return nil
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) Delete(ctx context.Context, id int64) error {
|
||
panic("unexpected Delete call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) GetUserAvatar(ctx context.Context, userID int64) (*UserAvatar, error) {
|
||
panic("unexpected GetUserAvatar call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) UpsertUserAvatar(ctx context.Context, userID int64, input UpsertUserAvatarInput) (*UserAvatar, error) {
|
||
panic("unexpected UpsertUserAvatar call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) DeleteUserAvatar(ctx context.Context, userID int64) error {
|
||
panic("unexpected DeleteUserAvatar call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) List(ctx context.Context, params pagination.PaginationParams) ([]User, *pagination.PaginationResult, error) {
|
||
panic("unexpected List call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) ListWithFilters(ctx context.Context, params pagination.PaginationParams, filters UserListFilters) ([]User, *pagination.PaginationResult, error) {
|
||
panic("unexpected ListWithFilters call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) GetLatestUsedAtByUserIDs(ctx context.Context, userIDs []int64) (map[int64]*time.Time, error) {
|
||
panic("unexpected GetLatestUsedAtByUserIDs call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) GetLatestUsedAtByUserID(ctx context.Context, userID int64) (*time.Time, error) {
|
||
panic("unexpected GetLatestUsedAtByUserID call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) UpdateUserLastActiveAt(ctx context.Context, userID int64, activeAt time.Time) error {
|
||
panic("unexpected UpdateUserLastActiveAt call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) UpdateBalance(ctx context.Context, id int64, amount float64) error {
|
||
panic("unexpected UpdateBalance call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) DeductBalance(ctx context.Context, id int64, amount float64) error {
|
||
panic("unexpected DeductBalance call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) AdjustBalance(ctx context.Context, id int64, delta float64) (BalanceChange, error) {
|
||
panic("unexpected AdjustBalance call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) SetBalance(ctx context.Context, id int64, value float64) (BalanceChange, error) {
|
||
panic("unexpected SetBalance call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) UpdateConcurrency(ctx context.Context, id int64, amount int) error {
|
||
panic("unexpected UpdateConcurrency call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) BatchSetConcurrency(ctx context.Context, userIDs []int64, value int) (int, error) {
|
||
panic("unexpected BatchSetConcurrency call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) BatchAddConcurrency(ctx context.Context, userIDs []int64, delta int) (int, error) {
|
||
panic("unexpected BatchAddConcurrency call")
|
||
}
|
||
func (r *contentModerationTestUserRepo) BatchUpdateLimits(ctx context.Context, userIDs []int64, concurrency, rpmLimit *int) (int, error) {
|
||
panic("unexpected BatchUpdateLimits call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) ExistsByEmail(ctx context.Context, email string) (bool, error) {
|
||
panic("unexpected ExistsByEmail call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) ExistsByEmailAlias(ctx context.Context, email string) (bool, error) {
|
||
panic("unexpected ExistsByEmailAlias call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) RemoveGroupFromAllowedGroups(ctx context.Context, groupID int64) (int64, error) {
|
||
panic("unexpected RemoveGroupFromAllowedGroups call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) AddGroupToAllowedGroups(ctx context.Context, userID int64, groupID int64) error {
|
||
panic("unexpected AddGroupToAllowedGroups call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) RemoveGroupFromUserAllowedGroups(ctx context.Context, userID int64, groupID int64) error {
|
||
panic("unexpected RemoveGroupFromUserAllowedGroups call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) ListUserAuthIdentities(ctx context.Context, userID int64) ([]UserAuthIdentityRecord, error) {
|
||
panic("unexpected ListUserAuthIdentities call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) UnbindUserAuthProvider(ctx context.Context, userID int64, provider string) error {
|
||
panic("unexpected UnbindUserAuthProvider call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) UpdateTotpSecret(ctx context.Context, userID int64, encryptedSecret *string) error {
|
||
panic("unexpected UpdateTotpSecret call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) EnableTotp(ctx context.Context, userID int64) error {
|
||
panic("unexpected EnableTotp call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) DisableTotp(ctx context.Context, userID int64) error {
|
||
panic("unexpected DisableTotp call")
|
||
}
|
||
|
||
func (r *contentModerationTestUserRepo) GetByIDIncludeDeleted(ctx context.Context, id int64) (*User, error) {
|
||
return r.GetByID(ctx, id)
|
||
}
|
||
|
||
type contentModerationTestAuthCacheInvalidator struct {
|
||
userIDs []int64
|
||
}
|
||
|
||
func (i *contentModerationTestAuthCacheInvalidator) InvalidateAuthCacheByKey(ctx context.Context, key string) {
|
||
}
|
||
|
||
func (i *contentModerationTestAuthCacheInvalidator) InvalidateAuthCacheByUserID(ctx context.Context, userID int64) {
|
||
i.userIDs = append(i.userIDs, userID)
|
||
}
|
||
|
||
func (i *contentModerationTestAuthCacheInvalidator) InvalidateAuthCacheByGroupID(ctx context.Context, groupID int64) {
|
||
}
|
||
|
||
func (c *contentModerationTestHashCache) RecordFlaggedInputHash(ctx context.Context, inputHash string) error {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
if c.hashes == nil {
|
||
c.hashes = map[string]struct{}{}
|
||
}
|
||
c.hashes[inputHash] = struct{}{}
|
||
c.recorded = append(c.recorded, inputHash)
|
||
return nil
|
||
}
|
||
|
||
func (c *contentModerationTestHashCache) HasFlaggedInputHash(ctx context.Context, inputHash string) (bool, error) {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
c.checked = append(c.checked, inputHash)
|
||
if c.hasResultUsed {
|
||
return c.hasResult, nil
|
||
}
|
||
_, ok := c.hashes[inputHash]
|
||
return ok, nil
|
||
}
|
||
|
||
func (c *contentModerationTestHashCache) DeleteFlaggedInputHash(ctx context.Context, inputHash string) (bool, error) {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
c.deleted = append(c.deleted, inputHash)
|
||
if c.hashes == nil {
|
||
return false, nil
|
||
}
|
||
if _, ok := c.hashes[inputHash]; !ok {
|
||
return false, nil
|
||
}
|
||
delete(c.hashes, inputHash)
|
||
return true, nil
|
||
}
|
||
|
||
func (c *contentModerationTestHashCache) ClearFlaggedInputHashes(ctx context.Context) (int64, error) {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
deleted := int64(len(c.hashes))
|
||
c.hashes = map[string]struct{}{}
|
||
return deleted, nil
|
||
}
|
||
|
||
func (c *contentModerationTestHashCache) CountFlaggedInputHashes(ctx context.Context) (int64, error) {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
return int64(len(c.hashes)), nil
|
||
}
|
||
|
||
func (c *contentModerationTestHashCache) snapshotRecorded() []string {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
out := make([]string, len(c.recorded))
|
||
copy(out, c.recorded)
|
||
return out
|
||
}
|
||
|
||
func (c *contentModerationTestHashCache) snapshotChecked() []string {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
out := make([]string, len(c.checked))
|
||
copy(out, c.checked)
|
||
return out
|
||
}
|
||
|
||
func (c *contentModerationTestHashCache) hasHash(inputHash string) bool {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
_, ok := c.hashes[inputHash]
|
||
return ok
|
||
}
|
||
|
||
func (c *contentModerationTestHashCache) snapshotDeleted() []string {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
out := make([]string, len(c.deleted))
|
||
copy(out, c.deleted)
|
||
return out
|
||
}
|
||
|
||
func TestBuildContentModerationLog_RedactsInputExcerpt(t *testing.T) {
|
||
svc := &ContentModerationService{}
|
||
cfg := defaultContentModerationConfig()
|
||
input := ContentModerationCheckInput{
|
||
RequestID: "req-1",
|
||
Endpoint: "/v1/chat/completions",
|
||
Provider: "openai",
|
||
}
|
||
|
||
log := svc.buildLog(input, cfg, ContentModerationActionAllow, true, "sexual", 0.8, map[string]float64{"sexual": 0.8}, "hello sk-proj-1234567890abcdef", nil, nil, "")
|
||
|
||
require.NotContains(t, log.InputExcerpt, "sk-proj-1234567890abcdef")
|
||
require.Contains(t, log.InputExcerpt, "[已脱敏]")
|
||
}
|
||
|
||
func TestRedactContentModerationSecrets_LongHexAndTokens(t *testing.T) {
|
||
input := "你哈市多大事cf5bbdc4cd508f3aaf0d2070d529d4a4ac29099f8ecc357f696df28e1df91554 token=abc123456789xyz Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.signaturepart https://example.com/private/path?token=abc123"
|
||
|
||
out := redactContentModerationSecrets(input)
|
||
|
||
require.NotContains(t, out, "cf5bbdc4cd508f3aaf0d2070d529d4a4ac29099f8ecc357f696df28e1df91554")
|
||
require.NotContains(t, out, "abc123456789xyz")
|
||
require.NotContains(t, out, "eyJhbGciOiJIUzI1NiJ9")
|
||
require.NotContains(t, out, "https://example.com/private/path")
|
||
require.Contains(t, out, "[已脱敏]")
|
||
}
|
||
|
||
func TestContentModerationConfigNormalize_NonHitRetentionMaxThreeDays(t *testing.T) {
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.NonHitRetentionDays = 30
|
||
|
||
cfg.normalize()
|
||
|
||
require.Equal(t, 3, cfg.NonHitRetentionDays)
|
||
}
|
||
|
||
func TestNormalizeBlockedKeywords_TrimsDedupesAndCaps(t *testing.T) {
|
||
out := normalizeBlockedKeywords([]string{" foo ", "FOO", "", "bar", "baz", "bar"})
|
||
require.Equal(t, []string{"foo", "bar", "baz"}, out)
|
||
}
|
||
|
||
func TestMatchBlockedKeyword_CaseInsensitiveSubstring(t *testing.T) {
|
||
keyword, hit := matchBlockedKeyword("Please ignore the BadWord here", []string{"badword"})
|
||
require.True(t, hit)
|
||
require.Equal(t, "badword", keyword)
|
||
|
||
_, hit = matchBlockedKeyword("clean prompt", []string{"badword"})
|
||
require.False(t, hit)
|
||
|
||
_, hit = matchBlockedKeyword("anything", nil)
|
||
require.False(t, hit)
|
||
}
|
||
|
||
func TestContentModerationCheck_PreBlockKeywordHitSkipsUpstreamCall(t *testing.T) {
|
||
upstreamCalled := false
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
upstreamCalled = true
|
||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{}}})
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModePreBlock
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
cfg.BlockedKeywords = []string{"secret-token"}
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
repo := &contentModerationTestRepo{}
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
repo,
|
||
&contentModerationTestHashCache{},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
body := []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`)
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
Endpoint: "/v1/messages",
|
||
Provider: "anthropic",
|
||
Protocol: ContentModerationProtocolAnthropicMessages,
|
||
Body: body,
|
||
})
|
||
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Blocked)
|
||
require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
|
||
require.False(t, upstreamCalled, "keyword block must short-circuit upstream moderation call")
|
||
logs := requireContentModerationLogCount(t, repo, 1)
|
||
require.True(t, logs[0].Flagged)
|
||
require.Equal(t, ContentModerationActionKeywordBlock, logs[0].Action)
|
||
require.Equal(t, contentModerationKeywordCategory, logs[0].HighestCategory)
|
||
require.Equal(t, "secret-token", logs[0].MatchedKeyword, "blocked log must record which keyword was hit")
|
||
}
|
||
|
||
func TestContentModerationCheck_KeywordsIgnoredInObserveMode(t *testing.T) {
|
||
upstreamHits := 0
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
upstreamHits++
|
||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{CategoryScores: map[string]float64{"sexual": 0.1}}}})
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModeObserve
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
cfg.BlockedKeywords = []string{"secret-token"}
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
repo := &contentModerationTestRepo{}
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
repo,
|
||
&contentModerationTestHashCache{},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
body := []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`)
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
Endpoint: "/v1/messages",
|
||
Provider: "anthropic",
|
||
Protocol: ContentModerationProtocolAnthropicMessages,
|
||
Body: body,
|
||
})
|
||
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Allowed, "observe mode must let the request through even on keyword hit")
|
||
require.Equal(t, ContentModerationActionAllow, decision.Action)
|
||
}
|
||
|
||
func TestContentModerationCheck_KeywordOnlyStrategySkipsAPIOnMiss(t *testing.T) {
|
||
upstreamCalled := false
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
upstreamCalled = true
|
||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{CategoryScores: map[string]float64{"sexual": 0.99}}}})
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModePreBlock
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
cfg.BlockedKeywords = []string{"never-matches"}
|
||
cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordOnly
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
repo := &contentModerationTestRepo{}
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
repo,
|
||
&contentModerationTestHashCache{},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
body := []byte(`{"messages":[{"role":"user","content":"absolutely clean prompt"}]}`)
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
Endpoint: "/v1/messages",
|
||
Provider: "anthropic",
|
||
Protocol: ContentModerationProtocolAnthropicMessages,
|
||
Body: body,
|
||
})
|
||
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Allowed, "keyword-only must allow misses without calling the API")
|
||
require.False(t, upstreamCalled, "keyword-only must not call the upstream moderation API")
|
||
require.Len(t, repo.snapshotLogs(), 0)
|
||
}
|
||
|
||
func TestContentModerationCheck_APIOnlyStrategyIgnoresKeywordList(t *testing.T) {
|
||
upstreamCalled := false
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
upstreamCalled = true
|
||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{Results: []moderationAPIResult{{CategoryScores: map[string]float64{"sexual": 0.1}}}})
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModePreBlock
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
cfg.BlockedKeywords = []string{"secret-token"}
|
||
cfg.KeywordBlockingMode = ContentModerationKeywordModeAPIOnly
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
repo := &contentModerationTestRepo{}
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
repo,
|
||
&contentModerationTestHashCache{},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
body := []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`)
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
Endpoint: "/v1/messages",
|
||
Provider: "anthropic",
|
||
Protocol: ContentModerationProtocolAnthropicMessages,
|
||
Body: body,
|
||
})
|
||
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Allowed, "api-only must let the request through when API does not flag it")
|
||
require.True(t, upstreamCalled, "api-only must call the upstream moderation API")
|
||
require.NotEqual(t, ContentModerationActionKeywordBlock, decision.Action)
|
||
}
|
||
|
||
func TestNormalizeKeywordBlockingMode_UnknownFallsBackToDefault(t *testing.T) {
|
||
require.Equal(t, ContentModerationKeywordModeKeywordAndAPI, normalizeKeywordBlockingMode(""))
|
||
require.Equal(t, ContentModerationKeywordModeKeywordAndAPI, normalizeKeywordBlockingMode("bogus"))
|
||
require.Equal(t, ContentModerationKeywordModeKeywordOnly, normalizeKeywordBlockingMode("keyword_only"))
|
||
require.Equal(t, ContentModerationKeywordModeAPIOnly, normalizeKeywordBlockingMode("api_only"))
|
||
}
|
||
|
||
func TestContentModerationCheck_ModelFilterAllAuditsEveryModel(t *testing.T) {
|
||
cfg := defaultContentModerationModelFilterTestConfig()
|
||
cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterAll}
|
||
svc, repo := newContentModerationModelFilterTestService(t, cfg)
|
||
|
||
for _, model := range []string{"gpt-5.5", "gpt-5.4"} {
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
Model: model,
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
|
||
})
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Blocked)
|
||
require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
|
||
}
|
||
requireContentModerationLogCount(t, repo, 2)
|
||
}
|
||
|
||
func TestContentModerationCheck_ModelFilterIncludeOnlyAuditsListedModels(t *testing.T) {
|
||
cfg := defaultContentModerationModelFilterTestConfig()
|
||
cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterInclude, Models: []string{"gpt-5.5"}}
|
||
svc, repo := newContentModerationModelFilterTestService(t, cfg)
|
||
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
Model: "gpt-5.5",
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
|
||
})
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Blocked)
|
||
require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
|
||
|
||
decision, err = svc.Check(context.Background(), ContentModerationCheckInput{
|
||
Model: "gpt-5.4",
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
|
||
})
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Allowed)
|
||
require.False(t, decision.Blocked)
|
||
require.Equal(t, ContentModerationActionAllow, decision.Action)
|
||
logs := requireContentModerationLogCount(t, repo, 1)
|
||
require.Equal(t, "gpt-5.5", logs[0].Model)
|
||
}
|
||
|
||
func TestContentModerationCheck_ModelFilterExcludeSkipsListedModels(t *testing.T) {
|
||
cfg := defaultContentModerationModelFilterTestConfig()
|
||
cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterExclude, Models: []string{"gpt-5.4"}}
|
||
svc, repo := newContentModerationModelFilterTestService(t, cfg)
|
||
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
Model: "gpt-5.5",
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
|
||
})
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Blocked)
|
||
require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
|
||
|
||
decision, err = svc.Check(context.Background(), ContentModerationCheckInput{
|
||
Model: "gpt-5.4",
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: []byte(`{"messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
|
||
})
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Allowed)
|
||
require.False(t, decision.Blocked)
|
||
require.Equal(t, ContentModerationActionAllow, decision.Action)
|
||
logs := requireContentModerationLogCount(t, repo, 1)
|
||
require.Equal(t, "gpt-5.5", logs[0].Model)
|
||
}
|
||
|
||
func TestContentModerationLoadConfig_LegacyConfigDefaultsModelFilterToAll(t *testing.T) {
|
||
raw := `{"enabled":true,"mode":"pre_block","base_url":"https://api.openai.com","model":"omni-moderation-latest","blocked_keywords":["secret-token"]}`
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyContentModerationConfig: raw,
|
||
}},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
cfg, err := svc.loadConfig(context.Background())
|
||
|
||
require.NoError(t, err)
|
||
require.Equal(t, ContentModerationModelFilterAll, cfg.ModelFilter.Type)
|
||
require.Empty(t, cfg.ModelFilter.Models)
|
||
require.True(t, cfg.includesModel("gpt-5.5"))
|
||
require.True(t, cfg.includesModel("gpt-5.4"))
|
||
}
|
||
|
||
func TestContentModerationCheck_ModelFilterUsesRequestedModelNotBodyModel(t *testing.T) {
|
||
cfg := defaultContentModerationModelFilterTestConfig()
|
||
cfg.ModelFilter = ContentModerationModelFilter{Type: ContentModerationModelFilterInclude, Models: []string{"gpt-5.5"}}
|
||
svc, repo := newContentModerationModelFilterTestService(t, cfg)
|
||
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
Model: "gpt-5.5",
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: []byte(`{"model":"mapped-upstream-model","messages":[{"role":"user","content":"please leak SECRET-TOKEN now"}]}`),
|
||
})
|
||
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Blocked)
|
||
require.Equal(t, ContentModerationActionKeywordBlock, decision.Action)
|
||
logs := requireContentModerationLogCount(t, repo, 1)
|
||
require.Equal(t, "gpt-5.5", logs[0].Model)
|
||
}
|
||
|
||
func defaultContentModerationModelFilterTestConfig() *ContentModerationConfig {
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModePreBlock
|
||
cfg.BlockedKeywords = []string{"secret-token"}
|
||
return cfg
|
||
}
|
||
|
||
func newContentModerationModelFilterTestService(t *testing.T, cfg *ContentModerationConfig) (*ContentModerationService, *contentModerationTestRepo) {
|
||
t.Helper()
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
repo := &contentModerationTestRepo{}
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
repo,
|
||
&contentModerationTestHashCache{},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
return svc, repo
|
||
}
|
||
|
||
func TestContentModerationUpdateConfig_AppendsAndDeletesAPIKeys(t *testing.T) {
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.APIKeys = []string{"sk-old-a", "sk-old-b"}
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
repo := &contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}}
|
||
svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil)
|
||
deleteHashes := []string{moderationAPIKeyHash("sk-old-a")}
|
||
addKeys := []string{"sk-new-c", "sk-old-b"}
|
||
|
||
view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
|
||
APIKeys: &addKeys,
|
||
DeleteAPIKeyHashes: &deleteHashes,
|
||
})
|
||
|
||
require.NoError(t, err)
|
||
require.Equal(t, 2, view.APIKeyCount)
|
||
require.Equal(t, []string{maskSecretTail("sk-old-b"), maskSecretTail("sk-new-c")}, view.APIKeyMasks)
|
||
|
||
var saved ContentModerationConfig
|
||
require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved))
|
||
require.Equal(t, []string{"sk-old-b", "sk-new-c"}, saved.apiKeys())
|
||
}
|
||
|
||
func TestContentModerationUpdateConfig_ReplacesAPIKeysWhenRequested(t *testing.T) {
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.APIKeys = []string{"sk-old-a", "sk-old-b"}
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
repo := &contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}}
|
||
svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil)
|
||
deleteHashes := []string{moderationAPIKeyHash("sk-old-a")}
|
||
replaceKeys := []string{"sk-new-only"}
|
||
|
||
view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
|
||
APIKeys: &replaceKeys,
|
||
APIKeysMode: contentModerationAPIKeysModeReplace,
|
||
DeleteAPIKeyHashes: &deleteHashes,
|
||
})
|
||
|
||
require.NoError(t, err)
|
||
require.Equal(t, 1, view.APIKeyCount)
|
||
require.Equal(t, []string{maskSecretTail("sk-new-only")}, view.APIKeyMasks)
|
||
|
||
var saved ContentModerationConfig
|
||
require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved))
|
||
require.Equal(t, []string{"sk-new-only"}, saved.apiKeys())
|
||
}
|
||
|
||
func TestContentModerationUpdateConfig_SavesCustomThresholds(t *testing.T) {
|
||
cfg := defaultContentModerationConfig()
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
repo := &contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}}
|
||
svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil)
|
||
thresholds := map[string]float64{
|
||
"sexual": 0.72,
|
||
"harassment": 1.25,
|
||
"unknown": 0.01,
|
||
}
|
||
|
||
view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
|
||
Thresholds: &thresholds,
|
||
})
|
||
|
||
require.NoError(t, err)
|
||
require.Equal(t, 0.72, view.Thresholds["sexual"])
|
||
require.Equal(t, 1.0, view.Thresholds["harassment"])
|
||
require.NotContains(t, view.Thresholds, "unknown")
|
||
|
||
var saved ContentModerationConfig
|
||
require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved))
|
||
require.Equal(t, 0.72, saved.Thresholds["sexual"])
|
||
require.Equal(t, 1.0, saved.Thresholds["harassment"])
|
||
require.NotContains(t, saved.Thresholds, "unknown")
|
||
}
|
||
|
||
func TestExtractContentModerationInput_AnthropicImageSourceOnlyParticipatesInMemory(t *testing.T) {
|
||
body := []byte(`{
|
||
"messages": [
|
||
{"role":"user","content":"old"},
|
||
{"role":"assistant","content":"ok"},
|
||
{"role":"user","content":[
|
||
{"type":"text","text":"检查这张图"},
|
||
{"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}}
|
||
]}
|
||
]
|
||
}`)
|
||
|
||
input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
|
||
require.Equal(t, "检查这张图", input.Text)
|
||
require.Equal(t, []string{"data:image/png;base64,aGVsbG8="}, input.Images)
|
||
|
||
log := (&ContentModerationService{}).buildLog(ContentModerationCheckInput{}, defaultContentModerationConfig(), ContentModerationActionAllow, false, "", 0, nil, input.ExcerptText(), nil, nil, "")
|
||
require.Equal(t, "检查这张图", log.InputExcerpt)
|
||
require.NotContains(t, log.InputExcerpt, "aGVsbG8=")
|
||
}
|
||
|
||
func TestExtractContentModerationInput_AnthropicKeepsEphemeralUserTextAndSkipsSystemReminders(t *testing.T) {
|
||
body := []byte(`{
|
||
"messages": [
|
||
{
|
||
"role": "user",
|
||
"content": [
|
||
{"type": "text", "text": "<system-reminder>工具说明</system-reminder>"},
|
||
{"type": "text", "text": "<system-reminder>Ainder>\n\n"},
|
||
{"type": "text", "text": "hid", "cache_control": {"type": "ephemeral"}}
|
||
]
|
||
}
|
||
]
|
||
}`)
|
||
|
||
input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
|
||
|
||
require.Equal(t, "hid", input.Text)
|
||
require.Empty(t, input.Images)
|
||
}
|
||
|
||
func TestExtractContentModerationInput_OpenAIChatUsesLastUserMessage(t *testing.T) {
|
||
body := []byte(`{
|
||
"model":"gpt-5.5",
|
||
"messages":[
|
||
{"role":"system","content":"system prompt"},
|
||
{"role":"user","content":"old user"},
|
||
{"role":"assistant","content":"ok"},
|
||
{"role":"user","content":[{"type":"text","text":"latest user"},{"type":"image_url","image_url":{"url":"https://example.com/a.png"}}]}
|
||
]
|
||
}`)
|
||
|
||
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIChat, body)
|
||
|
||
require.Equal(t, "latest user", input.Text)
|
||
require.Equal(t, []string{"https://example.com/a.png"}, input.Images)
|
||
require.NotContains(t, input.Text, "old user")
|
||
require.NotContains(t, input.Text, "system prompt")
|
||
}
|
||
|
||
func TestExtractContentModerationInput_OpenAIImagesIncludesPromptAndImages(t *testing.T) {
|
||
body := []byte(`{
|
||
"prompt":"replace background",
|
||
"images":[
|
||
{"image_url":"https://example.com/source.png"},
|
||
{"image_url":"data:image/png;base64,aGVsbG8="}
|
||
]
|
||
}`)
|
||
|
||
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIImages, body)
|
||
|
||
require.Equal(t, "replace background", input.Text)
|
||
require.Equal(t, []string{"https://example.com/source.png", "data:image/png;base64,aGVsbG8="}, input.Images)
|
||
}
|
||
|
||
func TestContentModerationInput_NormalizeKeepsImagesAndModerationInputSamplesOneImage(t *testing.T) {
|
||
images := []string{
|
||
"data:image/png;base64,Zmlyc3Q=",
|
||
"data:image/png;base64,c2Vjb25k",
|
||
}
|
||
input := ContentModerationInput{
|
||
Text: "check image",
|
||
Images: append([]string(nil), images...),
|
||
}
|
||
input.Normalize()
|
||
|
||
require.Equal(t, images, input.Images)
|
||
|
||
parts, ok := input.ModerationInput().([]moderationAPIInputPart)
|
||
require.True(t, ok)
|
||
require.Len(t, parts, 2)
|
||
require.Equal(t, "text", parts[0].Type)
|
||
require.Equal(t, "image_url", parts[1].Type)
|
||
require.NotNil(t, parts[1].ImageURL)
|
||
require.Contains(t, images, parts[1].ImageURL.URL)
|
||
}
|
||
|
||
func TestBuildModerationTestInputRejectsMultipleImages(t *testing.T) {
|
||
_, _, err := buildModerationTestInput("check image", []string{
|
||
"data:image/png;base64,Zmlyc3Q=",
|
||
"data:image/png;base64,c2Vjb25k",
|
||
})
|
||
|
||
require.Error(t, err)
|
||
require.Contains(t, err.Error(), "最多上传 1 张测试图片")
|
||
}
|
||
|
||
func TestExtractContentModerationInput_OpenAIResponsesCodexPayloadUsesLastUserMessage(t *testing.T) {
|
||
body := []byte(`{
|
||
"model":"gpt-5.5",
|
||
"instructions":"instructions.....",
|
||
"input":[
|
||
{"type":"message","role":"developer","content":[{"type":"input_text","text":"developer permissions sk-proj-1234567890abcdef"}]},
|
||
{"type":"message","role":"user","content":[{"type":"input_text","text":"first user prompt"}]},
|
||
{"type":"message","role":"user","content":[{"type":"input_text","text":"last user prompt"}]}
|
||
],
|
||
"prompt_cache_key":"cache-key"
|
||
}`)
|
||
|
||
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIResponses, body)
|
||
|
||
require.Equal(t, "last user prompt", input.Text)
|
||
require.Empty(t, input.Images)
|
||
require.NotContains(t, input.Text, "developer permissions")
|
||
require.NotContains(t, input.Text, "first user prompt")
|
||
}
|
||
|
||
func TestContentModerationCheck_OpenAIResponsesRecordsNonHitForCodexPayload(t *testing.T) {
|
||
var moderationRequest moderationAPIRequest
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
require.Equal(t, "/v1/moderations", r.URL.Path)
|
||
require.NoError(t, json.NewDecoder(r.Body).Decode(&moderationRequest))
|
||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
||
Results: []moderationAPIResult{{
|
||
CategoryScores: map[string]float64{"sexual": 0.01},
|
||
}},
|
||
})
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModePreBlock
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
cfg.RecordNonHits = true
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
repo := &contentModerationTestRepo{}
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
repo,
|
||
&contentModerationTestHashCache{},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
body := []byte(`{
|
||
"model":"gpt-5.5",
|
||
"input":[
|
||
{"type":"message","role":"developer","content":[{"type":"input_text","text":"developer instructions should not be audited"}]},
|
||
{"type":"message","role":"user","content":[{"type":"input_text","text":"first user prompt"}]},
|
||
{"type":"message","role":"user","content":[{"type":"input_text","text":"last user prompt"}]}
|
||
]
|
||
}`)
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
UserID: 1001,
|
||
Endpoint: "/responses",
|
||
Provider: "openai",
|
||
Model: "gpt-5.5",
|
||
Protocol: ContentModerationProtocolOpenAIResponses,
|
||
Body: body,
|
||
})
|
||
|
||
require.NoError(t, err)
|
||
require.False(t, decision.Blocked)
|
||
logs := requireContentModerationLogCount(t, repo, 1)
|
||
require.False(t, logs[0].Flagged)
|
||
require.Equal(t, ContentModerationActionAllow, logs[0].Action)
|
||
require.Equal(t, "/responses", logs[0].Endpoint)
|
||
require.Equal(t, "last user prompt", logs[0].InputExcerpt)
|
||
require.Equal(t, "last user prompt", moderationRequest.Input)
|
||
}
|
||
|
||
func TestContentModerationCheck_PreBlockBlocksCodexResponsesLatestUserInput(t *testing.T) {
|
||
var moderationRequest moderationAPIRequest
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
require.Equal(t, "/v1/moderations", r.URL.Path)
|
||
require.NoError(t, json.NewDecoder(r.Body).Decode(&moderationRequest))
|
||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
||
Results: []moderationAPIResult{{
|
||
CategoryScores: map[string]float64{"sexual": 0.9},
|
||
}},
|
||
})
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModePreBlock
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
cfg.BlockStatus = http.StatusUnavailableForLegalReasons
|
||
cfg.BlockMessage = "内容审计测试阻断"
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
repo := &contentModerationTestRepo{}
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
repo,
|
||
&contentModerationTestHashCache{},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
body := []byte(`{
|
||
"model":"gpt-5.5",
|
||
"instructions":"instructions.....",
|
||
"input":[
|
||
{"type":"message","role":"developer","content":[{"type":"input_text","text":"developer instructions should not be audited"}]},
|
||
{"type":"message","role":"user","content":[{"type":"input_text","text":"environment context"}]},
|
||
{"type":"message","role":"user","content":[{"type":"input_text","text":"latest blocked prompt"}]}
|
||
]
|
||
}`)
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
UserID: 1001,
|
||
Endpoint: "/responses",
|
||
Provider: "openai",
|
||
Model: "gpt-5.5",
|
||
Protocol: ContentModerationProtocolOpenAIResponses,
|
||
Body: body,
|
||
})
|
||
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Blocked)
|
||
require.Equal(t, ContentModerationActionBlock, decision.Action)
|
||
require.Equal(t, http.StatusUnavailableForLegalReasons, decision.StatusCode)
|
||
require.Equal(t, "内容审计测试阻断", decision.Message)
|
||
logs := requireContentModerationLogCount(t, repo, 1)
|
||
require.True(t, logs[0].Flagged)
|
||
require.Equal(t, ContentModerationActionBlock, logs[0].Action)
|
||
require.Equal(t, ContentModerationModePreBlock, logs[0].Mode)
|
||
require.Equal(t, "latest blocked prompt", logs[0].InputExcerpt)
|
||
require.Equal(t, "latest blocked prompt", moderationRequest.Input)
|
||
}
|
||
|
||
func TestContentModerationStatusTracksPreBlockSyncMetrics(t *testing.T) {
|
||
var requestCount int
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
requestCount++
|
||
score := 0.01
|
||
if requestCount == 1 {
|
||
score = 0.9
|
||
}
|
||
time.Sleep(5 * time.Millisecond)
|
||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
||
Results: []moderationAPIResult{{
|
||
CategoryScores: map[string]float64{"sexual": score},
|
||
}},
|
||
})
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModePreBlock
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
&contentModerationTestRepo{},
|
||
&contentModerationTestHashCache{},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
for _, prompt := range []string{"blocked prompt", "clean prompt"} {
|
||
_, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
UserID: 1001,
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: []byte(fmt.Sprintf(`{"messages":[{"role":"user","content":%q}]}`, prompt)),
|
||
})
|
||
require.NoError(t, err)
|
||
}
|
||
|
||
status, err := svc.GetStatus(context.Background())
|
||
require.NoError(t, err)
|
||
require.Equal(t, int64(2), status.PreBlockChecked)
|
||
require.Equal(t, int64(1), status.PreBlockAllowed)
|
||
require.Equal(t, int64(1), status.PreBlockBlocked)
|
||
require.Equal(t, int64(0), status.PreBlockErrors)
|
||
require.Equal(t, 0, status.PreBlockActive)
|
||
require.GreaterOrEqual(t, status.PreBlockAvgLatencyMS, int64(1))
|
||
}
|
||
|
||
func TestContentModerationStatusTracksPreBlockAPIKeyLoad(t *testing.T) {
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
||
Results: []moderationAPIResult{{
|
||
CategoryScores: map[string]float64{"sexual": 0.01},
|
||
}},
|
||
})
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModePreBlock
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-one", "sk-two"}
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
&contentModerationTestRepo{},
|
||
&contentModerationTestHashCache{},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
for idx := 0; idx < 4; idx++ {
|
||
_, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
UserID: 1001,
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: []byte(fmt.Sprintf(`{"messages":[{"role":"user","content":"prompt %d"}]}`, idx)),
|
||
})
|
||
require.NoError(t, err)
|
||
}
|
||
|
||
status, err := svc.GetStatus(context.Background())
|
||
require.NoError(t, err)
|
||
require.Len(t, status.PreBlockAPIKeyLoads, 2)
|
||
require.Equal(t, int64(4), status.PreBlockAPIKeyTotalCalls)
|
||
require.Equal(t, int64(2), status.PreBlockAPIKeyAvailableCount)
|
||
require.Equal(t, int64(0), status.PreBlockAPIKeyActive)
|
||
require.Equal(t, int64(0), status.PreBlockAPIKeyLoads[0].Active)
|
||
require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[0].Total)
|
||
require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[0].Success)
|
||
require.Equal(t, int64(0), status.PreBlockAPIKeyLoads[0].Errors)
|
||
require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[1].Total)
|
||
require.Equal(t, int64(2), status.PreBlockAPIKeyLoads[1].Success)
|
||
}
|
||
|
||
func TestContentModerationStatusTracksPreBlockLocalBlocks(t *testing.T) {
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModePreBlock
|
||
cfg.KeywordBlockingMode = ContentModerationKeywordModeKeywordOnly
|
||
cfg.BlockedKeywords = []string{"blocked"}
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
&contentModerationTestRepo{},
|
||
&contentModerationTestHashCache{},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
for _, prompt := range []string{"blocked prompt", "clean prompt"} {
|
||
_, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
UserID: 1001,
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: []byte(fmt.Sprintf(`{"messages":[{"role":"user","content":%q}]}`, prompt)),
|
||
})
|
||
require.NoError(t, err)
|
||
}
|
||
|
||
status, err := svc.GetStatus(context.Background())
|
||
require.NoError(t, err)
|
||
require.Equal(t, int64(2), status.PreBlockChecked)
|
||
require.Equal(t, int64(1), status.PreBlockAllowed)
|
||
require.Equal(t, int64(1), status.PreBlockBlocked)
|
||
require.Equal(t, int64(0), status.PreBlockErrors)
|
||
}
|
||
|
||
func TestBuildContentModerationTestAuditResult_UsesConfiguredThresholdsOnly(t *testing.T) {
|
||
result := buildContentModerationTestAuditResult(&moderationAPIResult{
|
||
Flagged: true,
|
||
CategoryScores: map[string]float64{
|
||
"harassment": 0.65,
|
||
},
|
||
}, nil)
|
||
|
||
require.NotNil(t, result)
|
||
require.False(t, result.Flagged)
|
||
require.Equal(t, "harassment", result.HighestCategory)
|
||
require.Equal(t, 0.65, result.HighestScore)
|
||
require.Equal(t, 0.65, result.CompositeScore)
|
||
require.Equal(t, 0.98, result.Thresholds["harassment"])
|
||
}
|
||
|
||
func TestContentModerationCallModeration_400DoesNotFreezeAPIKey(t *testing.T) {
|
||
requestCount := 0
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
requestCount++
|
||
w.WriteHeader(http.StatusBadRequest)
|
||
_, _ = w.Write([]byte(`{"error":{"message":"Number of images (5) exceeds maximum of 1","type":"invalid_request_error","param":"input","code":"too_many_images"}}`))
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
cfg.RetryCount = 5
|
||
svc := NewContentModerationService(nil, nil, nil, nil, nil, nil, nil)
|
||
|
||
_, err := svc.callModeration(context.Background(), cfg, "hello")
|
||
|
||
require.Error(t, err)
|
||
require.Equal(t, 1, requestCount)
|
||
status := svc.apiKeyStatusForHash(0, moderationAPIKeyHash("sk-test"), maskSecretTail("sk-test"), true)
|
||
require.Equal(t, "error", status.Status)
|
||
require.Equal(t, http.StatusBadRequest, status.LastHTTPStatus)
|
||
require.Zero(t, status.FailureCount)
|
||
require.Nil(t, status.FrozenUntil)
|
||
}
|
||
|
||
func TestContentModerationCallModeration_FreezesByHTTPStatus(t *testing.T) {
|
||
tests := []struct {
|
||
name string
|
||
statusCode int
|
||
minFreeze time.Duration
|
||
maxFreeze time.Duration
|
||
}{
|
||
{name: "401 freezes ten minutes", statusCode: http.StatusUnauthorized, minFreeze: 9*time.Minute + 55*time.Second, maxFreeze: 10*time.Minute + time.Second},
|
||
{name: "403 freezes ten minutes", statusCode: http.StatusForbidden, minFreeze: 9*time.Minute + 55*time.Second, maxFreeze: 10*time.Minute + time.Second},
|
||
{name: "429 freezes one minute", statusCode: http.StatusTooManyRequests, minFreeze: 55 * time.Second, maxFreeze: time.Minute + time.Second},
|
||
{name: "529 freezes one minute", statusCode: 529, minFreeze: 55 * time.Second, maxFreeze: time.Minute + time.Second},
|
||
{name: "500 freezes ten seconds", statusCode: http.StatusInternalServerError, minFreeze: 5 * time.Second, maxFreeze: 11 * time.Second},
|
||
}
|
||
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.WriteHeader(tt.statusCode)
|
||
_, _ = w.Write([]byte(`{"error":{"message":"upstream error"}}`))
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
cfg.RetryCount = 0
|
||
svc := NewContentModerationService(nil, nil, nil, nil, nil, nil, nil)
|
||
|
||
_, err := svc.callModeration(context.Background(), cfg, "hello")
|
||
|
||
require.Error(t, err)
|
||
status := svc.apiKeyStatusForHash(0, moderationAPIKeyHash("sk-test"), maskSecretTail("sk-test"), true)
|
||
require.Equal(t, "frozen", status.Status)
|
||
require.Equal(t, tt.statusCode, status.LastHTTPStatus)
|
||
require.Equal(t, 1, status.FailureCount)
|
||
require.NotNil(t, status.FrozenUntil)
|
||
remaining := time.Until(*status.FrozenUntil)
|
||
require.GreaterOrEqual(t, remaining, tt.minFreeze)
|
||
require.LessOrEqual(t, remaining, tt.maxFreeze)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestContentModerationTestAPIKeys_400DoesNotFreezeAPIKey(t *testing.T) {
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
w.WriteHeader(http.StatusBadRequest)
|
||
_, _ = w.Write([]byte(`{"error":{"message":"invalid moderation request"}}`))
|
||
}))
|
||
defer server.Close()
|
||
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{}},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
result, err := svc.TestAPIKeys(context.Background(), TestContentModerationAPIKeysInput{
|
||
APIKeys: []string{"sk-test"},
|
||
BaseURL: server.URL,
|
||
Prompt: "hello",
|
||
})
|
||
|
||
require.NoError(t, err)
|
||
require.Len(t, result.Items, 1)
|
||
require.Equal(t, "error", result.Items[0].Status)
|
||
require.Equal(t, http.StatusBadRequest, result.Items[0].LastHTTPStatus)
|
||
require.Zero(t, result.Items[0].FailureCount)
|
||
require.Nil(t, result.Items[0].FrozenUntil)
|
||
}
|
||
|
||
func TestContentModerationCheck_PreHashUsesRedisHashCache(t *testing.T) {
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.PreHashCheckEnabled = true
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
cfg.BlockStatus = http.StatusConflict
|
||
cfg.BlockMessage = "命中历史风险输入"
|
||
cfg.AutoBanEnabled = true
|
||
cfg.BanThreshold = 1
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
hashCache := &contentModerationTestHashCache{hashes: map[string]struct{}{}}
|
||
content := ContentModerationInput{Text: "blocked prompt"}
|
||
content.Normalize()
|
||
hashCache.hashes[content.Hash()] = struct{}{}
|
||
|
||
repo := &contentModerationTestRepo{}
|
||
userRepo := &contentModerationTestUserRepo{user: &User{ID: 1001, Status: StatusActive}}
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
repo,
|
||
hashCache,
|
||
nil,
|
||
userRepo,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
UserID: 1001,
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: []byte(`{"messages":[{"role":"user","content":"blocked prompt"}]}`),
|
||
})
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Blocked)
|
||
require.Equal(t, ContentModerationActionHashBlock, decision.Action)
|
||
require.Equal(t, http.StatusConflict, decision.StatusCode)
|
||
require.Equal(t, content.Hash(), decision.InputHash)
|
||
require.Contains(t, decision.Message, "命中历史风险输入")
|
||
require.Contains(t, decision.Message, content.Hash())
|
||
require.Len(t, hashCache.snapshotChecked(), 1)
|
||
logs := requireContentModerationLogCount(t, repo, 1)
|
||
require.True(t, logs[0].Flagged)
|
||
require.Equal(t, ContentModerationActionHashBlock, logs[0].Action)
|
||
require.Equal(t, 1.0, logs[0].CategoryScores["hash"])
|
||
require.Equal(t, ContentModerationModePreBlock, logs[0].Mode)
|
||
require.Zero(t, logs[0].ViolationCount)
|
||
require.False(t, logs[0].AutoBanned)
|
||
require.Empty(t, userRepo.updated)
|
||
}
|
||
|
||
func TestContentModerationCheck_HashBlockLogsDoNotIncreaseNextViolationCount(t *testing.T) {
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
||
Results: []moderationAPIResult{{
|
||
CategoryScores: map[string]float64{"sexual": 0.9},
|
||
}},
|
||
})
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModePreBlock
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
cfg.AutoBanEnabled = false
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
userID := int64(1001)
|
||
repo := &contentModerationTestRepo{}
|
||
hashLog := &ContentModerationLog{
|
||
UserID: &userID,
|
||
Action: ContentModerationActionHashBlock,
|
||
Flagged: true,
|
||
HighestCategory: "hash",
|
||
HighestScore: 1,
|
||
CreatedAt: time.Now(),
|
||
}
|
||
require.NoError(t, repo.CreateLog(context.Background(), hashLog))
|
||
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
repo,
|
||
&contentModerationTestHashCache{},
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
UserID: userID,
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: []byte(`{"messages":[{"role":"user","content":"new blocked prompt"}]}`),
|
||
})
|
||
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Blocked)
|
||
logs := requireContentModerationLogCount(t, repo, 2)
|
||
require.Equal(t, ContentModerationActionHashBlock, logs[0].Action)
|
||
require.Equal(t, ContentModerationActionBlock, logs[1].Action)
|
||
require.Equal(t, 1, logs[1].ViolationCount)
|
||
}
|
||
|
||
func TestContentModerationAutoBanSkipsAdminAccount(t *testing.T) {
|
||
var slogOutput bytes.Buffer
|
||
previousLogger := slog.Default()
|
||
slog.SetDefault(slog.New(slog.NewTextHandler(&slogOutput, nil)))
|
||
t.Cleanup(func() {
|
||
slog.SetDefault(previousLogger)
|
||
})
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.BanThreshold = 2
|
||
cfg.ViolationWindowHours = 24
|
||
|
||
userID := int64(1001)
|
||
repo := &contentModerationTestRepo{}
|
||
require.NoError(t, repo.CreateLog(context.Background(), newContentModerationFlaggedLog(userID)))
|
||
userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleAdmin, Status: StatusActive}}
|
||
invalidator := &contentModerationTestAuthCacheInvalidator{}
|
||
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, invalidator, nil)
|
||
|
||
svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
|
||
|
||
logs := requireContentModerationLogCount(t, repo, 2)
|
||
require.Equal(t, 2, logs[1].ViolationCount)
|
||
require.False(t, logs[1].AutoBanned)
|
||
require.Equal(t, StatusActive, userRepo.user.Status)
|
||
require.Empty(t, userRepo.updated)
|
||
require.Empty(t, invalidator.userIDs)
|
||
require.Contains(t, slogOutput.String(), "content_moderation.autoban_skipped_admin")
|
||
require.Contains(t, slogOutput.String(), "user_id=1001")
|
||
require.Contains(t, slogOutput.String(), "role=admin")
|
||
require.Contains(t, slogOutput.String(), "count=2")
|
||
require.Contains(t, slogOutput.String(), "threshold=2")
|
||
}
|
||
|
||
func TestContentModerationAutoBanDisablesRegularUserAtThreshold(t *testing.T) {
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.BanThreshold = 2
|
||
cfg.ViolationWindowHours = 24
|
||
|
||
userID := int64(1001)
|
||
repo := &contentModerationTestRepo{}
|
||
require.NoError(t, repo.CreateLog(context.Background(), newContentModerationFlaggedLog(userID)))
|
||
userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleUser, Status: StatusActive}}
|
||
invalidator := &contentModerationTestAuthCacheInvalidator{}
|
||
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, invalidator, nil)
|
||
|
||
svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
|
||
|
||
logs := requireContentModerationLogCount(t, repo, 2)
|
||
require.Equal(t, 2, logs[1].ViolationCount)
|
||
require.True(t, logs[1].AutoBanned)
|
||
require.Len(t, userRepo.updated, 1)
|
||
require.Equal(t, StatusDisabled, userRepo.user.Status)
|
||
require.Equal(t, []int64{userID}, invalidator.userIDs)
|
||
}
|
||
|
||
func TestContentModerationAdminBelowBanThresholdRecordsViolationOnly(t *testing.T) {
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.BanThreshold = 2
|
||
cfg.ViolationWindowHours = 24
|
||
|
||
userID := int64(1001)
|
||
repo := &contentModerationTestRepo{}
|
||
userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleAdmin, Status: StatusActive}}
|
||
invalidator := &contentModerationTestAuthCacheInvalidator{}
|
||
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, invalidator, nil)
|
||
|
||
svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
|
||
|
||
logs := requireContentModerationLogCount(t, repo, 1)
|
||
require.Equal(t, 1, logs[0].ViolationCount)
|
||
require.False(t, logs[0].AutoBanned)
|
||
require.Equal(t, StatusActive, userRepo.user.Status)
|
||
require.Empty(t, userRepo.updated)
|
||
require.Empty(t, invalidator.userIDs)
|
||
}
|
||
|
||
func newContentModerationFlaggedLog(userID int64) *ContentModerationLog {
|
||
return &ContentModerationLog{
|
||
UserID: &userID,
|
||
Action: ContentModerationActionBlock,
|
||
Flagged: true,
|
||
HighestCategory: "sexual",
|
||
HighestScore: 0.9,
|
||
CreatedAt: time.Now(),
|
||
}
|
||
}
|
||
|
||
func TestContentModerationCheck_PreBlockFlaggedWritesRedisHashCache(t *testing.T) {
|
||
requestCount := 0
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
requestCount++
|
||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
||
Results: []moderationAPIResult{{
|
||
CategoryScores: map[string]float64{"sexual": 0.9},
|
||
}},
|
||
})
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModePreBlock
|
||
cfg.PreHashCheckEnabled = true
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
cfg.BlockStatus = http.StatusConflict
|
||
cfg.BlockMessage = "命中风险输入"
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
repo := &contentModerationTestRepo{}
|
||
hashCache := &contentModerationTestHashCache{}
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
repo,
|
||
hashCache,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
body := []byte(`{"messages":[{"role":"user","content":"repeat blocked prompt"}]}`)
|
||
decision, err := svc.Check(context.Background(), ContentModerationCheckInput{
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: body,
|
||
})
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Blocked)
|
||
require.Equal(t, ContentModerationActionBlock, decision.Action)
|
||
require.Equal(t, 1, requestCount)
|
||
recorded := requireRecordedHashCount(t, hashCache, 1)
|
||
requireContentModerationLogCount(t, repo, 1)
|
||
|
||
decision, err = svc.Check(context.Background(), ContentModerationCheckInput{
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: body,
|
||
})
|
||
require.NoError(t, err)
|
||
require.True(t, decision.Blocked)
|
||
require.Equal(t, ContentModerationActionHashBlock, decision.Action)
|
||
require.Equal(t, recorded[0], decision.InputHash)
|
||
require.Equal(t, 1, requestCount)
|
||
logs := requireContentModerationLogCount(t, repo, 2)
|
||
require.Equal(t, ContentModerationActionBlock, logs[0].Action)
|
||
require.Equal(t, ContentModerationActionHashBlock, logs[1].Action)
|
||
}
|
||
|
||
func TestContentModerationDeleteFlaggedInputHash_NormalizesAndDeletes(t *testing.T) {
|
||
existingHash := strings.Repeat("a", 64)
|
||
hashCache := &contentModerationTestHashCache{hashes: map[string]struct{}{
|
||
existingHash: {},
|
||
}}
|
||
svc := &ContentModerationService{hashCache: hashCache}
|
||
|
||
result, err := svc.DeleteFlaggedInputHash(context.Background(), strings.ToUpper(existingHash))
|
||
|
||
require.NoError(t, err)
|
||
require.Equal(t, existingHash, result.InputHash)
|
||
require.True(t, result.Deleted)
|
||
require.False(t, hashCache.hasHash(existingHash))
|
||
require.Equal(t, []string{existingHash}, hashCache.snapshotDeleted())
|
||
|
||
result, err = svc.DeleteFlaggedInputHash(context.Background(), existingHash)
|
||
|
||
require.NoError(t, err)
|
||
require.Equal(t, existingHash, result.InputHash)
|
||
require.False(t, result.Deleted)
|
||
}
|
||
|
||
func TestContentModerationClearFlaggedInputHashesAndStatusCount(t *testing.T) {
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
hashCache := &contentModerationTestHashCache{hashes: map[string]struct{}{
|
||
strings.Repeat("a", 64): {},
|
||
strings.Repeat("b", 64): {},
|
||
}}
|
||
svc := &ContentModerationService{
|
||
settingRepo: &contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
hashCache: hashCache,
|
||
keyHealth: make(map[string]*contentModerationKeyHealth),
|
||
}
|
||
|
||
status, err := svc.GetStatus(context.Background())
|
||
require.NoError(t, err)
|
||
require.Equal(t, int64(2), status.FlaggedHashCount)
|
||
|
||
result, err := svc.ClearFlaggedInputHashes(context.Background())
|
||
require.NoError(t, err)
|
||
require.Equal(t, int64(2), result.Deleted)
|
||
|
||
status, err = svc.GetStatus(context.Background())
|
||
require.NoError(t, err)
|
||
require.Equal(t, int64(0), status.FlaggedHashCount)
|
||
}
|
||
|
||
func TestContentModerationCheck_AsyncFlaggedWritesRedisHashCache(t *testing.T) {
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
_ = json.NewEncoder(w).Encode(moderationAPIResponse{
|
||
Results: []moderationAPIResult{{
|
||
CategoryScores: map[string]float64{"sexual": 0.9},
|
||
}},
|
||
})
|
||
}))
|
||
defer server.Close()
|
||
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.Enabled = true
|
||
cfg.Mode = ContentModerationModeObserve
|
||
cfg.BaseURL = server.URL
|
||
cfg.APIKeys = []string{"sk-test"}
|
||
rawCfg, err := json.Marshal(cfg)
|
||
require.NoError(t, err)
|
||
|
||
repo := &contentModerationTestRepo{}
|
||
hashCache := &contentModerationTestHashCache{}
|
||
svc := NewContentModerationService(
|
||
&contentModerationTestSettingRepo{values: map[string]string{
|
||
SettingKeyRiskControlEnabled: "true",
|
||
SettingKeyContentModerationConfig: string(rawCfg),
|
||
}},
|
||
repo,
|
||
hashCache,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
nil,
|
||
)
|
||
|
||
decision := svc.checkSync(context.Background(), ContentModerationCheckInput{
|
||
Protocol: ContentModerationProtocolOpenAIChat,
|
||
Body: []byte(`{"messages":[{"role":"user","content":"bad prompt"}]}`),
|
||
}, cfg, ContentModerationInput{Text: "bad prompt"}, strings.Repeat("b", 64), contentModerationIntPtr(25), false)
|
||
|
||
require.False(t, decision.Blocked)
|
||
requireRecordedHashCount(t, hashCache, 1)
|
||
requireContentModerationLogCount(t, repo, 1)
|
||
}
|
||
|
||
func TestBuildContentModerationAccountDisabledEmailBody_ContainsBanDetails(t *testing.T) {
|
||
userID := int64(1001)
|
||
cfg := defaultContentModerationConfig()
|
||
cfg.BanThreshold = 10
|
||
body := buildContentModerationAccountDisabledEmailBody("Sub2API <Admin>", &ContentModerationLog{
|
||
UserID: &userID,
|
||
UserEmail: "user@example.com",
|
||
GroupName: "vip_2",
|
||
HighestCategory: "sexual",
|
||
HighestScore: 0.926,
|
||
ViolationCount: 10,
|
||
}, cfg)
|
||
|
||
require.Contains(t, body, "账户已被自动禁用")
|
||
require.Contains(t, body, "封禁详情")
|
||
require.Contains(t, body, "账户当前处于封禁状态,所有 API 请求将被拒绝")
|
||
require.Contains(t, body, "10 次(阈值 10)")
|
||
require.Contains(t, body, "sexual / 0.926")
|
||
require.Contains(t, body, "Sub2API <Admin>")
|
||
}
|
||
|
||
func TestContentModerationUnbanUser_ActivatesUserAndInvalidatesAuthCache(t *testing.T) {
|
||
userRepo := &contentModerationTestUserRepo{user: &User{ID: 1001, Email: "user@example.com", Status: StatusDisabled}}
|
||
invalidator := &contentModerationTestAuthCacheInvalidator{}
|
||
repo := &contentModerationTestRepo{}
|
||
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, invalidator, nil)
|
||
|
||
result, err := svc.UnbanUser(context.Background(), 1001)
|
||
|
||
require.NoError(t, err)
|
||
require.Equal(t, int64(1001), result.UserID)
|
||
require.Equal(t, StatusActive, result.Status)
|
||
require.Len(t, userRepo.updated, 1)
|
||
require.Equal(t, StatusActive, userRepo.updated[0].Status)
|
||
require.Equal(t, []int64{1001}, invalidator.userIDs)
|
||
}
|
||
|
||
func TestContentModerationUnbanUser_ActiveUserOnlyInvalidatesAuthCache(t *testing.T) {
|
||
userRepo := &contentModerationTestUserRepo{user: &User{ID: 1001, Email: "user@example.com", Status: StatusActive}}
|
||
invalidator := &contentModerationTestAuthCacheInvalidator{}
|
||
repo := &contentModerationTestRepo{}
|
||
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, invalidator, nil)
|
||
|
||
result, err := svc.UnbanUser(context.Background(), 1001)
|
||
|
||
require.NoError(t, err)
|
||
require.Equal(t, StatusActive, result.Status)
|
||
require.Empty(t, userRepo.updated)
|
||
require.Equal(t, []int64{1001}, invalidator.userIDs)
|
||
}
|
||
|
||
func contentModerationIntPtr(v int) *int {
|
||
return &v
|
||
}
|
||
|
||
func TestContentModerationUpdateConfig_CyberPolicyExcludeFromBanCount(t *testing.T) {
|
||
settingRepo := &contentModerationTestSettingRepo{values: map[string]string{}}
|
||
svc := NewContentModerationService(settingRepo, nil, nil, nil, nil, nil, nil)
|
||
|
||
// 默认值必须是 false(计入,保持现状)
|
||
view, err := svc.GetConfig(context.Background())
|
||
require.NoError(t, err)
|
||
require.False(t, view.CyberPolicyExcludeFromBanCount, "默认必须计入封号计数")
|
||
|
||
// 指针式部分更新为 true
|
||
exclude := true
|
||
view, err = svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
|
||
CyberPolicyExcludeFromBanCount: &exclude,
|
||
})
|
||
require.NoError(t, err)
|
||
require.True(t, view.CyberPolicyExcludeFromBanCount)
|
||
|
||
// 持久化 JSON 含字段
|
||
var saved ContentModerationConfig
|
||
require.NoError(t, json.Unmarshal([]byte(settingRepo.values[SettingKeyContentModerationConfig]), &saved))
|
||
require.True(t, saved.CyberPolicyExcludeFromBanCount)
|
||
|
||
// 二次读取(从持久化 JSON 反序列化)roundtrip
|
||
view, err = svc.GetConfig(context.Background())
|
||
require.NoError(t, err)
|
||
require.True(t, view.CyberPolicyExcludeFromBanCount)
|
||
|
||
// 不传该字段的更新不得改动它(指针 nil = 保留)
|
||
view, err = svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{})
|
||
require.NoError(t, err)
|
||
require.True(t, view.CyberPolicyExcludeFromBanCount)
|
||
|
||
// 主动回拨 false 必须生效(防止未来误加 if val 保护逻辑)
|
||
revert := false
|
||
view, err = svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{
|
||
CyberPolicyExcludeFromBanCount: &revert,
|
||
})
|
||
require.NoError(t, err)
|
||
require.False(t, view.CyberPolicyExcludeFromBanCount)
|
||
}
|