mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-11-17 14:57:27 +00:00
merge: sync batch image foundation with upstream main
This commit is contained in:
commit
9703ca9d33
119 files changed
+7719
-1389
No files matched your search
@@ -482,7 +482,7 @@ func (r *accountRepository) List(ctx context.Context, params pagination.Paginati
|
||||
return r.ListWithFilters(ctx, params, "", "", "", "", 0, "")
|
||||
}
|
||||
|
||||
func (r *accountRepository) ListWithFilters(ctx context.Context, params pagination.PaginationParams, platform, accountType, status, search string, groupID int64, privacyMode string) ([]service.Account, *pagination.PaginationResult, error) {
|
||||
func (r *accountRepository) accountListFilteredQuery(platform, accountType, status, search string, groupID int64, privacyMode string) *dbent.AccountQuery {
|
||||
q := r.client.Account.Query()
|
||||
|
||||
if platform != "" {
|
||||
@@ -575,6 +575,11 @@ func (r *accountRepository) ListWithFilters(ctx context.Context, params paginati
|
||||
}))
|
||||
}
|
||||
|
||||
return q
|
||||
}
|
||||
|
||||
func (r *accountRepository) ListWithFilters(ctx context.Context, params pagination.PaginationParams, platform, accountType, status, search string, groupID int64, privacyMode string) ([]service.Account, *pagination.PaginationResult, error) {
|
||||
q := r.accountListFilteredQuery(platform, accountType, status, search, groupID, privacyMode)
|
||||
// Clone before Count so interceptor-appended predicates (SoftDeleteMixin's
|
||||
// deleted_at IS NULL) don't accumulate on the shared builder and pollute the
|
||||
// subsequent list query. Same pattern used in group_repo/promo_code_repo/user_repo
|
||||
@@ -603,6 +608,14 @@ func (r *accountRepository) ListWithFilters(ctx context.Context, params paginati
|
||||
return outAccounts, paginationResultFromTotal(int64(total), params), nil
|
||||
}
|
||||
|
||||
func (r *accountRepository) ListAllWithFilters(ctx context.Context, platform, accountType, status, search string, groupID int64, privacyMode string) ([]service.Account, error) {
|
||||
accounts, err := r.accountListFilteredQuery(platform, accountType, status, search, groupID, privacyMode).All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.accountsToService(ctx, accounts)
|
||||
}
|
||||
|
||||
func (r *accountRepository) ListOpsAccountsForStats(ctx context.Context, platformFilter string, groupIDFilter *int64) ([]service.Account, error) {
|
||||
if r == nil || r.client == nil {
|
||||
return []service.Account{}, nil
|
||||
@@ -1061,6 +1074,90 @@ func (r *accountRepository) ListSchedulableByGroupID(ctx context.Context, groupI
|
||||
})
|
||||
}
|
||||
|
||||
func (r *accountRepository) ListSchedulableCapacityByGroupIDs(ctx context.Context, groupIDs []int64) ([]service.GroupAccountCapacityRow, error) {
|
||||
groupIDs = uniquePositiveInt64s(groupIDs)
|
||||
if len(groupIDs) == 0 {
|
||||
return []service.GroupAccountCapacityRow{}, nil
|
||||
}
|
||||
if r.sql == nil {
|
||||
rows := make([]service.GroupAccountCapacityRow, 0)
|
||||
for _, groupID := range groupIDs {
|
||||
accounts, err := r.ListSchedulableByGroupID(ctx, groupID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range accounts {
|
||||
acc := &accounts[i]
|
||||
rows = append(rows, service.GroupAccountCapacityRow{
|
||||
GroupID: groupID,
|
||||
AccountID: acc.ID,
|
||||
Concurrency: acc.Concurrency,
|
||||
Extra: copyJSONMap(acc.Extra),
|
||||
SessionWindowStart: acc.SessionWindowStart,
|
||||
SessionWindowEnd: acc.SessionWindowEnd,
|
||||
SessionWindowStatus: acc.SessionWindowStatus,
|
||||
})
|
||||
}
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
rows, err := r.sql.QueryContext(ctx, `
|
||||
SELECT
|
||||
ag.group_id,
|
||||
a.id AS account_id,
|
||||
a.concurrency,
|
||||
COALESCE(a.extra, '{}'::jsonb)::text AS extra,
|
||||
a.session_window_start,
|
||||
a.session_window_end,
|
||||
COALESCE(a.session_window_status, '') AS session_window_status
|
||||
FROM account_groups ag
|
||||
JOIN accounts a ON a.id = ag.account_id
|
||||
WHERE ag.group_id = ANY($1)
|
||||
AND a.deleted_at IS NULL
|
||||
AND a.status = $2
|
||||
AND a.schedulable = TRUE
|
||||
AND (a.temp_unschedulable_until IS NULL OR a.temp_unschedulable_until <= $3)
|
||||
AND (a.expires_at IS NULL OR a.expires_at > $3 OR a.auto_pause_on_expired = FALSE)
|
||||
AND (a.overload_until IS NULL OR a.overload_until <= $3)
|
||||
AND (a.rate_limit_reset_at IS NULL OR a.rate_limit_reset_at <= $3)
|
||||
ORDER BY ag.group_id ASC, ag.priority ASC, a.priority ASC, a.id ASC
|
||||
`, pq.Array(groupIDs), service.StatusActive, time.Now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
out := make([]service.GroupAccountCapacityRow, 0)
|
||||
for rows.Next() {
|
||||
var row service.GroupAccountCapacityRow
|
||||
var extraRaw string
|
||||
if err := rows.Scan(
|
||||
&row.GroupID,
|
||||
&row.AccountID,
|
||||
&row.Concurrency,
|
||||
&extraRaw,
|
||||
&row.SessionWindowStart,
|
||||
&row.SessionWindowEnd,
|
||||
&row.SessionWindowStatus,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if extraRaw != "" && extraRaw != "null" {
|
||||
var extra map[string]any
|
||||
if err := json.Unmarshal([]byte(extraRaw), &extra); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.Extra = extra
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *accountRepository) ListSchedulableByPlatform(ctx context.Context, platform string) ([]service.Account, error) {
|
||||
now := time.Now()
|
||||
accounts, err := r.client.Account.Query().
|
||||
|
||||
@@ -174,6 +174,29 @@ var (
|
||||
return 1
|
||||
`)
|
||||
|
||||
// cleanupExpiredSlotKeysScript 批量清理实际存在的账号槽位键,避免后台任务从数据库加载全量账号。
|
||||
// KEYS = 有序集合键列表,ARGV[1] = TTL(秒)。
|
||||
cleanupExpiredSlotKeysScript = redis.NewScript(`
|
||||
-- Redis 3.2-4.x compat: opt into effects replication so redis.call('TIME')
|
||||
-- replicates correctly. No-op on Redis 5.0+ (effects replication is default).
|
||||
redis.replicate_commands()
|
||||
local ttl = tonumber(ARGV[1])
|
||||
local timeResult = redis.call('TIME')
|
||||
local now = tonumber(timeResult[1])
|
||||
local expireBefore = now - ttl
|
||||
local removed = 0
|
||||
for i = 1, #KEYS do
|
||||
local key = KEYS[i]
|
||||
removed = removed + redis.call('ZREMRANGEBYSCORE', key, '-inf', expireBefore)
|
||||
if redis.call('ZCARD', key) == 0 then
|
||||
redis.call('DEL', key)
|
||||
else
|
||||
redis.call('EXPIRE', key, ttl)
|
||||
end
|
||||
end
|
||||
return removed
|
||||
`)
|
||||
|
||||
// startupCleanupScript 清理非当前进程前缀的槽位成员。
|
||||
// KEYS 是有序集合键列表,ARGV[1] 是当前进程前缀,ARGV[2] 是槽位 TTL。
|
||||
// 遍历每个 KEYS[i],移除前缀不匹配的成员,清空后删 key,否则刷新 EXPIRE。
|
||||
@@ -503,6 +526,10 @@ func (c *concurrencyCache) CleanupExpiredAccountSlots(ctx context.Context, accou
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *concurrencyCache) CleanupExpiredAccountSlotKeys(ctx context.Context) error {
|
||||
return c.cleanupExpiredSlotKeysByPattern(ctx, accountSlotKeyPrefix+"*")
|
||||
}
|
||||
|
||||
func (c *concurrencyCache) CleanupStaleProcessSlots(ctx context.Context, activeRequestPrefix string) error {
|
||||
if activeRequestPrefix == "" {
|
||||
return nil
|
||||
@@ -527,6 +554,29 @@ func (c *concurrencyCache) CleanupStaleProcessSlots(ctx context.Context, activeR
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupExpiredSlotKeysByPattern 扫描实际存在的账号槽位键并批量清理过期成员。
|
||||
func (c *concurrencyCache) cleanupExpiredSlotKeysByPattern(ctx context.Context, pattern string) error {
|
||||
const scanCount = 200
|
||||
var cursor uint64
|
||||
for {
|
||||
keys, nextCursor, err := c.rdb.Scan(ctx, cursor, pattern, scanCount).Result()
|
||||
if err != nil {
|
||||
return fmt.Errorf("scan %s: %w", pattern, err)
|
||||
}
|
||||
if len(keys) > 0 {
|
||||
_, err := cleanupExpiredSlotKeysScript.Run(ctx, c.rdb, keys, c.slotTTLSeconds).Result()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleanup expired slots %s: %w", pattern, err)
|
||||
}
|
||||
}
|
||||
cursor = nextCursor
|
||||
if cursor == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupSlotsByPattern 扫描匹配 pattern 的有序集合键,批量调用 Lua 脚本清理非当前进程成员。
|
||||
func (c *concurrencyCache) cleanupSlotsByPattern(ctx context.Context, pattern, activePrefix string) error {
|
||||
const scanCount = 200
|
||||
|
||||
@@ -435,6 +435,39 @@ func (s *ConcurrencyCacheSuite) TestCleanupExpiredAccountSlots_NoExpired() {
|
||||
require.Equal(s.T(), 2, cur)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestCleanupExpiredAccountSlotKeys() {
|
||||
now := time.Now().Unix()
|
||||
expiredTime := now - int64(testSlotTTL.Seconds()) - 10
|
||||
accountKeyWithFresh := fmt.Sprintf("%s%d", accountSlotKeyPrefix, 301)
|
||||
accountKeyExpiredOnly := fmt.Sprintf("%s%d", accountSlotKeyPrefix, 302)
|
||||
userKey := fmt.Sprintf("%s%d", userSlotKeyPrefix, 303)
|
||||
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountKeyWithFresh,
|
||||
redis.Z{Score: float64(expiredTime), Member: "expired"},
|
||||
redis.Z{Score: float64(now), Member: "fresh"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, accountKeyExpiredOnly,
|
||||
redis.Z{Score: float64(expiredTime), Member: "expired-only"},
|
||||
).Err())
|
||||
require.NoError(s.T(), s.rdb.ZAdd(s.ctx, userKey,
|
||||
redis.Z{Score: float64(expiredTime), Member: "user-expired"},
|
||||
).Err())
|
||||
|
||||
require.NoError(s.T(), s.cache.CleanupExpiredAccountSlotKeys(s.ctx))
|
||||
|
||||
accountMembers, err := s.rdb.ZRange(s.ctx, accountKeyWithFresh, 0, -1).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), []string{"fresh"}, accountMembers)
|
||||
|
||||
exists, err := s.rdb.Exists(s.ctx, accountKeyExpiredOnly).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.EqualValues(s.T(), 0, exists)
|
||||
|
||||
userMembers, err := s.rdb.ZRange(s.ctx, userKey, 0, -1).Result()
|
||||
require.NoError(s.T(), err)
|
||||
require.Equal(s.T(), []string{"user-expired"}, userMembers)
|
||||
}
|
||||
|
||||
func (s *ConcurrencyCacheSuite) TestCleanupStaleProcessSlots_RemovesOldPrefixesAndWaitCounters() {
|
||||
accountID := int64(901)
|
||||
userID := int64(902)
|
||||
|
||||
@@ -472,6 +472,49 @@ func (r *groupRepository) ListActive(ctx context.Context) ([]service.Group, erro
|
||||
return outGroups, nil
|
||||
}
|
||||
|
||||
func (r *groupRepository) ListActiveIDs(ctx context.Context) ([]int64, error) {
|
||||
if r.sql != nil {
|
||||
rows, err := r.sql.QueryContext(ctx, `
|
||||
SELECT id
|
||||
FROM groups
|
||||
WHERE status = $1
|
||||
AND deleted_at IS NULL
|
||||
ORDER BY sort_order ASC, id ASC
|
||||
`, service.StatusActive)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
ids := make([]int64, 0)
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
groups, err := r.client.Group.Query().
|
||||
Where(group.StatusEQ(service.StatusActive)).
|
||||
Select(group.FieldID).
|
||||
Order(dbent.Asc(group.FieldSortOrder), dbent.Asc(group.FieldID)).
|
||||
All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]int64, 0, len(groups))
|
||||
for i := range groups {
|
||||
ids = append(ids, groups[i].ID)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (r *groupRepository) ListActiveByPlatform(ctx context.Context, platform string) ([]service.Group, error) {
|
||||
groups, err := r.client.Group.Query().
|
||||
Where(group.StatusEQ(service.StatusActive), group.PlatformEQ(platform)).
|
||||
|
||||
@@ -85,10 +85,21 @@ func TestBuildOpsErrorLogsWhere_CyberPolicyStatusExemption(t *testing.T) {
|
||||
t.Fatalf("default filter must still include the status >= 400 guard for non-cyber rows\nfull: %s", where)
|
||||
}
|
||||
|
||||
// phase=upstream skips the status guard entirely — exemption is irrelevant there.
|
||||
// phase=upstream WITHOUT the recovered-upstream opt-in keeps the status guard:
|
||||
// request-error list endpoints filter by phase=upstream as a plain condition.
|
||||
whereUpstream, _ := buildOpsErrorLogsWhere(&service.OpsErrorLogFilter{Phase: "upstream"})
|
||||
if strings.Contains(whereUpstream, "status_code") {
|
||||
t.Fatalf("upstream phase filter must not add any status_code clause\nfull: %s", whereUpstream)
|
||||
if !strings.Contains(whereUpstream, "COALESCE(e.status_code, 0) >= 400") {
|
||||
t.Fatalf("upstream phase without IncludeRecoveredUpstream must keep the status guard\nfull: %s", whereUpstream)
|
||||
}
|
||||
if !strings.Contains(whereUpstream, "e.error_phase = $") {
|
||||
t.Fatalf("upstream phase filter must emit the error_phase condition\nfull: %s", whereUpstream)
|
||||
}
|
||||
|
||||
// phase=upstream WITH IncludeRecoveredUpstream (ops 上游列表) skips the guard,
|
||||
// exposing recovered (<400) upstream rows.
|
||||
whereRecovered, _ := buildOpsErrorLogsWhere(&service.OpsErrorLogFilter{Phase: "upstream", IncludeRecoveredUpstream: true})
|
||||
if strings.Contains(whereRecovered, "status_code") {
|
||||
t.Fatalf("upstream phase with IncludeRecoveredUpstream must not add any status_code clause\nfull: %s", whereRecovered)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -177,6 +177,37 @@ func opsInsertErrorLogArgs(input *service.OpsInsertErrorLogInput) []any {
|
||||
}
|
||||
}
|
||||
|
||||
// opsErrorLogsOrderBy builds the ORDER BY clause from a whitelist, mirroring
|
||||
// usageLogOrderBy semantics. Unknown SortBy falls back to created_at; e.id is
|
||||
// always appended as tiebreaker for stable pagination.
|
||||
func opsErrorLogsOrderBy(filter *service.OpsErrorLogFilter) string {
|
||||
sortBy := ""
|
||||
sortOrder := ""
|
||||
if filter != nil {
|
||||
sortBy = strings.ToLower(strings.TrimSpace(filter.SortBy))
|
||||
sortOrder = strings.ToLower(strings.TrimSpace(filter.SortOrder))
|
||||
}
|
||||
|
||||
var column string
|
||||
switch sortBy {
|
||||
case "model":
|
||||
column = "COALESCE(NULLIF(TRIM(e.requested_model), ''), e.model)"
|
||||
case "status_code":
|
||||
// 与展示列/过滤保持同义:列表展示 COALESCE(upstream_status_code, status_code, 0),
|
||||
// status_code 过滤也用同一表达式,故排序必须一致——否则 recovered upstream 行
|
||||
//(status_code<400 但展示上游 5xx)排序键与显示值/分页切分不符。
|
||||
column = "COALESCE(e.upstream_status_code, e.status_code, 0)"
|
||||
default:
|
||||
column = "e.created_at"
|
||||
}
|
||||
|
||||
dir := "DESC"
|
||||
if sortOrder == "asc" {
|
||||
dir = "ASC"
|
||||
}
|
||||
return fmt.Sprintf("%s %s, e.id %s", column, dir, dir)
|
||||
}
|
||||
|
||||
func (r *opsRepository) ListErrorLogs(ctx context.Context, filter *service.OpsErrorLogFilter) (*service.OpsErrorLogList, error) {
|
||||
if r == nil || r.db == nil {
|
||||
return nil, fmt.Errorf("nil ops repository")
|
||||
@@ -233,25 +264,29 @@ SELECT
|
||||
COALESCE(a.name, ''),
|
||||
e.group_id,
|
||||
COALESCE(g.name, ''),
|
||||
CASE WHEN e.client_ip IS NULL THEN NULL ELSE e.client_ip::text END,
|
||||
CASE WHEN e.client_ip IS NULL THEN NULL ELSE host(e.client_ip) END,
|
||||
COALESCE(e.request_path, ''),
|
||||
e.stream,
|
||||
COALESCE(e.inbound_endpoint, ''),
|
||||
COALESCE(e.upstream_endpoint, ''),
|
||||
COALESCE(e.requested_model, ''),
|
||||
COALESCE(e.upstream_model, ''),
|
||||
COALESCE(e.user_agent, ''),
|
||||
e.request_type,
|
||||
COALESCE(ak.name, ''),
|
||||
ak.deleted_at,
|
||||
COALESCE(e.deleted_key_name, '')
|
||||
COALESCE(e.deleted_key_name, ''),
|
||||
e.deleted_key_owner_user_id,
|
||||
COALESCE(du.email, '')
|
||||
FROM ops_error_logs e
|
||||
LEFT JOIN accounts a ON e.account_id = a.id
|
||||
LEFT JOIN groups g ON e.group_id = g.id
|
||||
LEFT JOIN users u ON e.user_id = u.id
|
||||
LEFT JOIN users u2 ON e.resolved_by_user_id = u2.id
|
||||
LEFT JOIN users du ON e.deleted_key_owner_user_id = du.id
|
||||
LEFT JOIN api_keys ak ON ak.id = e.api_key_id
|
||||
` + where + `
|
||||
ORDER BY e.created_at DESC
|
||||
ORDER BY ` + opsErrorLogsOrderBy(filter) + `
|
||||
LIMIT $` + itoa(len(args)+1) + ` OFFSET $` + itoa(len(args)+2)
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, selectSQL, argsWithLimit...)
|
||||
@@ -279,6 +314,8 @@ LIMIT $` + itoa(len(args)+1) + ` OFFSET $` + itoa(len(args)+2)
|
||||
var apiKeyName string
|
||||
var apiKeyDeletedAt sql.NullTime
|
||||
var deletedKeyName string
|
||||
var deletedKeyOwnerID sql.NullInt64
|
||||
var deletedKeyOwnerEmail string
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.CreatedAt,
|
||||
@@ -311,10 +348,13 @@ LIMIT $` + itoa(len(args)+1) + ` OFFSET $` + itoa(len(args)+2)
|
||||
&item.UpstreamEndpoint,
|
||||
&item.RequestedModel,
|
||||
&item.UpstreamModel,
|
||||
&item.UserAgent,
|
||||
&requestType,
|
||||
&apiKeyName,
|
||||
&apiKeyDeletedAt,
|
||||
&deletedKeyName,
|
||||
&deletedKeyOwnerID,
|
||||
&deletedKeyOwnerEmail,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -364,6 +404,12 @@ LIMIT $` + itoa(len(args)+1) + ` OFFSET $` + itoa(len(args)+2)
|
||||
}
|
||||
// 已删除:ak.deleted_at 非空(软删),或仅命中 deleted_key_name 兜底。
|
||||
item.APIKeyDeleted = apiKeyDeletedAt.Valid || (apiKeyName == "" && deletedKeyName != "")
|
||||
// 已删除 KEY 所有者快照:认证失败行 user_id 为空,列表用户列以此回退。
|
||||
if deletedKeyOwnerID.Valid {
|
||||
v := deletedKeyOwnerID.Int64
|
||||
item.DeletedKeyOwnerUserID = &v
|
||||
item.DeletedKeyOwnerEmail = deletedKeyOwnerEmail
|
||||
}
|
||||
out = append(out, &item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
@@ -417,7 +463,7 @@ SELECT
|
||||
COALESCE(a.name, ''),
|
||||
e.group_id,
|
||||
COALESCE(g.name, ''),
|
||||
CASE WHEN e.client_ip IS NULL THEN NULL ELSE e.client_ip::text END,
|
||||
CASE WHEN e.client_ip IS NULL THEN NULL ELSE host(e.client_ip) END,
|
||||
COALESCE(e.request_path, ''),
|
||||
e.stream,
|
||||
COALESCE(e.inbound_endpoint, ''),
|
||||
@@ -927,12 +973,14 @@ func buildOpsErrorLogsWhere(filter *service.OpsErrorLogFilter) (string, []any) {
|
||||
if filter != nil {
|
||||
resolvedFilter = filter.Resolved
|
||||
}
|
||||
// Keep list endpoints scoped to client errors unless explicitly filtering upstream phase.
|
||||
// Keep list endpoints scoped to client errors unless the caller explicitly opts
|
||||
// into recovered upstream rows (Phase=="upstream" + IncludeRecoveredUpstream,
|
||||
// ops 专用上游列表)。请求错误语义的端点即便过滤 phase=upstream 也保留该守卫。
|
||||
// cyber_policy is exempt from the status >= 400 guard: streaming cyber hits arrive with
|
||||
// status 200 (the SSE stream opened successfully before upstream returned response.failed),
|
||||
// but they are always client-visible blocked requests that belong in admin + user error
|
||||
// lists. Without the exemption the entire streaming-path cyber sink would be invisible.
|
||||
if phaseFilter != "upstream" {
|
||||
if phaseFilter != "upstream" || filter == nil || !filter.IncludeRecoveredUpstream {
|
||||
clauses = append(clauses, "(COALESCE(e.status_code, 0) >= 400 OR e.error_type = 'cyber_policy')")
|
||||
}
|
||||
|
||||
|
||||
@@ -518,7 +518,7 @@ func filterSchedulerCredentials(credentials map[string]any) map[string]any {
|
||||
if len(credentials) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := []string{"model_mapping", "compact_model_mapping", "api_key", "project_id", "oauth_type"}
|
||||
keys := []string{"model_mapping", "compact_model_mapping", "api_key", "project_id", "oauth_type", "plan_type"}
|
||||
filtered := make(map[string]any)
|
||||
for _, key := range keys {
|
||||
if value, ok := credentials[key]; ok && value != nil {
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFilterSchedulerCredentialsKeepsSubscriptionPlanType(t *testing.T) {
|
||||
filtered := filterSchedulerCredentials(map[string]any{
|
||||
"plan_type": "plus",
|
||||
"access_token": "secret-access-token",
|
||||
"refresh_token": "secret-refresh-token",
|
||||
})
|
||||
|
||||
require.Equal(t, "plus", filtered["plan_type"])
|
||||
require.NotContains(t, filtered, "access_token")
|
||||
require.NotContains(t, filtered, "refresh_token")
|
||||
}
|
||||
|
||||
func TestSchedulerMetadataAccountKeepsOpenAISubscriptionIdentity(t *testing.T) {
|
||||
account := service.Account{
|
||||
ID: 24,
|
||||
Platform: service.PlatformOpenAI,
|
||||
Type: service.AccountTypeOAuth,
|
||||
Credentials: map[string]any{
|
||||
"plan_type": "plus",
|
||||
"access_token": "secret-access-token",
|
||||
},
|
||||
}
|
||||
|
||||
metadata := buildSchedulerMetadataAccount(account)
|
||||
|
||||
require.True(t, metadata.IsOpenAIChatGPTSubscription())
|
||||
require.Empty(t, metadata.GetCredential("access_token"))
|
||||
}
|
||||
@@ -372,12 +372,13 @@ func (r *usageLogRepository) CreateBestEffort(ctx context.Context, log *service.
|
||||
}
|
||||
}
|
||||
|
||||
// 队列满时阻塞等待而非立即丢弃:批处理器持续排空队列,短暂等待即可入队。
|
||||
// 立即丢弃会造成“已扣费但无 usage_log”的永久数据缺口(issue #3656);
|
||||
// 阻塞上限由调用方 ctx 期限约束,超时后由上层同步兜底。
|
||||
select {
|
||||
case r.bestEffortBatchCh <- req:
|
||||
case <-ctx.Done():
|
||||
return service.MarkUsageLogCreateDropped(ctx.Err())
|
||||
default:
|
||||
return service.MarkUsageLogCreateDropped(errors.New("usage log best-effort queue full"))
|
||||
}
|
||||
|
||||
select {
|
||||
@@ -493,12 +494,12 @@ func (r *usageLogRepository) createBatched(ctx context.Context, log *service.Usa
|
||||
resultCh: make(chan usageLogCreateResult, 1),
|
||||
}
|
||||
|
||||
// 队列满时阻塞等待而非立即报错:本路径是 best-effort 丢弃后的最后兜底,
|
||||
// 立即失败会让日志永久丢失;阻塞上限由调用方 ctx 期限约束。
|
||||
select {
|
||||
case r.createBatchCh <- req:
|
||||
case <-ctx.Done():
|
||||
return false, service.MarkUsageLogCreateNotPersisted(ctx.Err())
|
||||
default:
|
||||
return false, service.MarkUsageLogCreateNotPersisted(errors.New("usage log create batch queue full"))
|
||||
}
|
||||
|
||||
select {
|
||||
@@ -520,22 +521,28 @@ func (r *usageLogRepository) createBatched(ctx context.Context, log *service.Usa
|
||||
}
|
||||
|
||||
func (r *usageLogRepository) ensureCreateBatcher() {
|
||||
if r == nil || r.db == nil || r.createBatchCh != nil {
|
||||
if r == nil || r.db == nil {
|
||||
return
|
||||
}
|
||||
// nil 检查必须在 Once 内部:在外层做无同步快路径读会与 Once 内的写构成数据竞争。
|
||||
r.createBatchOnce.Do(func() {
|
||||
r.createBatchCh = make(chan usageLogCreateRequest, usageLogCreateBatchQueueCap)
|
||||
go r.runCreateBatcher(r.db)
|
||||
if r.createBatchCh == nil {
|
||||
r.createBatchCh = make(chan usageLogCreateRequest, usageLogCreateBatchQueueCap)
|
||||
go r.runCreateBatcher(r.db)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (r *usageLogRepository) ensureBestEffortBatcher() {
|
||||
if r == nil || r.db == nil || r.bestEffortBatchCh != nil {
|
||||
if r == nil || r.db == nil {
|
||||
return
|
||||
}
|
||||
// 同 ensureCreateBatcher:nil 检查放在 Once 内部以避免数据竞争。
|
||||
r.bestEffortBatchOnce.Do(func() {
|
||||
r.bestEffortBatchCh = make(chan usageLogBestEffortRequest, usageLogBestEffortBatchQueueCap)
|
||||
go r.runBestEffortBatcher(r.db)
|
||||
if r.bestEffortBatchCh == nil {
|
||||
r.bestEffortBatchCh = make(chan usageLogBestEffortRequest, usageLogBestEffortBatchQueueCap)
|
||||
go r.runBestEffortBatcher(r.db)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -288,21 +288,21 @@ func TestUsageLogRepositoryCreateBestEffort_BatchPathDuplicateRequestID(t *testi
|
||||
}, 3*time.Second, 20*time.Millisecond)
|
||||
}
|
||||
|
||||
func TestUsageLogRepositoryCreateBestEffort_QueueFullReturnsDropped(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
func TestUsageLogRepositoryCreateBestEffort_QueueFullBlocksUntilCtxDeadline(t *testing.T) {
|
||||
// 队列满时不再立即丢弃:阻塞等待入队,直到调用方 ctx 到期才标记 dropped(issue #3656)。
|
||||
client := testEntClient(t)
|
||||
repo := newUsageLogRepositoryWithSQL(client, integrationDB)
|
||||
repo.bestEffortBatchCh = make(chan usageLogBestEffortRequest, 1)
|
||||
repo.bestEffortBatchCh <- usageLogBestEffortRequest{}
|
||||
|
||||
user := mustCreateUser(t, client, &service.User{Email: fmt.Sprintf("usage-best-effort-full-%d@example.com", time.Now().UnixNano())})
|
||||
apiKey := mustCreateApiKey(t, client, &service.APIKey{UserID: user.ID, Key: "sk-usage-best-effort-full-" + uuid.NewString(), Name: "k"})
|
||||
account := mustCreateAccount(t, client, &service.Account{Name: "acc-usage-best-effort-full-" + uuid.NewString()})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
err := repo.CreateBestEffort(ctx, &service.UsageLog{
|
||||
UserID: user.ID,
|
||||
APIKeyID: apiKey.ID,
|
||||
AccountID: account.ID,
|
||||
UserID: 1,
|
||||
APIKeyID: 2,
|
||||
AccountID: 3,
|
||||
RequestID: uuid.NewString(),
|
||||
Model: "claude-3",
|
||||
InputTokens: 10,
|
||||
@@ -314,6 +314,40 @@ func TestUsageLogRepositoryCreateBestEffort_QueueFullReturnsDropped(t *testing.T
|
||||
|
||||
require.Error(t, err)
|
||||
require.True(t, service.IsUsageLogCreateDropped(err))
|
||||
require.GreaterOrEqual(t, time.Since(start), 150*time.Millisecond)
|
||||
}
|
||||
|
||||
func TestUsageLogRepositoryCreateBestEffort_QueueFullWaitsForDrain(t *testing.T) {
|
||||
// 队列满但批处理器随后排空时,阻塞的入队应成功完成而非丢弃。
|
||||
client := testEntClient(t)
|
||||
repo := newUsageLogRepositoryWithSQL(client, integrationDB)
|
||||
repo.bestEffortBatchCh = make(chan usageLogBestEffortRequest, 1)
|
||||
repo.bestEffortBatchCh <- usageLogBestEffortRequest{}
|
||||
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
<-repo.bestEffortBatchCh // 排空占位请求,为阻塞中的入队腾出空间
|
||||
req := <-repo.bestEffortBatchCh
|
||||
sendUsageLogBestEffortResult(req.resultCh, nil)
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := repo.CreateBestEffort(ctx, &service.UsageLog{
|
||||
UserID: 1,
|
||||
APIKeyID: 2,
|
||||
AccountID: 3,
|
||||
RequestID: uuid.NewString(),
|
||||
Model: "claude-3",
|
||||
InputTokens: 10,
|
||||
OutputTokens: 20,
|
||||
TotalCost: 0.5,
|
||||
ActualCost: 0.5,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestUsageLogRepositoryCreate_BatchPathCanceledContextMarksNotPersisted(t *testing.T) {
|
||||
@@ -346,7 +380,7 @@ func TestUsageLogRepositoryCreate_BatchPathCanceledContextMarksNotPersisted(t *t
|
||||
}
|
||||
|
||||
func TestUsageLogRepositoryCreate_BatchPathQueueFullMarksNotPersisted(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
// 队列满时阻塞等待入队,直到调用方 ctx 到期才标记 not persisted(issue #3656)。
|
||||
client := testEntClient(t)
|
||||
repo := newUsageLogRepositoryWithSQL(client, integrationDB)
|
||||
repo.createBatchCh = make(chan usageLogCreateRequest, 1)
|
||||
@@ -356,6 +390,10 @@ func TestUsageLogRepositoryCreate_BatchPathQueueFullMarksNotPersisted(t *testing
|
||||
apiKey := mustCreateApiKey(t, client, &service.APIKey{UserID: user.ID, Key: "sk-usage-create-full-" + uuid.NewString(), Name: "k"})
|
||||
account := mustCreateAccount(t, client, &service.Account{Name: "acc-usage-create-full-" + uuid.NewString()})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
inserted, err := repo.Create(ctx, &service.UsageLog{
|
||||
UserID: user.ID,
|
||||
APIKeyID: apiKey.ID,
|
||||
@@ -372,6 +410,7 @@ func TestUsageLogRepositoryCreate_BatchPathQueueFullMarksNotPersisted(t *testing
|
||||
require.False(t, inserted)
|
||||
require.Error(t, err)
|
||||
require.True(t, service.IsUsageLogCreateNotPersisted(err))
|
||||
require.GreaterOrEqual(t, time.Since(start), 150*time.Millisecond)
|
||||
}
|
||||
|
||||
func TestUsageLogRepositoryCreate_BatchPathCanceledAfterQueueMarksNotPersisted(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user