mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-11-17 14:57:27 +00:00
fix: prevent silent usage_logs drops under queue overflow (#3656)
高并发下扣费成功但 usage_logs 被双层异步队列静默丢弃(单用户实测丢失 49%),
导致消费排行与对账缺口。根因是 6a685727 为避免溢出时单行 INSERT 踩踏数据库,
把三处"队列满"改成了立即终态丢弃。
改为有界阻塞背压,兼顾数据不丢与批量写入模式不变:
- CreateBestEffort/createBatched 入队移除 default 立即丢弃分支,
队列满时阻塞等待(受 detached 15s ctx 期限约束)
- writeUsageLogBestEffort 收到 dropped 不再直接放弃,统一走
repo.Create 同步兜底(仍经批处理器;ctx 耗尽时换新 detached 窗口),
重复写入由 ON CONFLICT DO NOTHING 幂等防护
- worker 池默认溢出策略 sample→sync(viper 默认 + 池常量),
溢出时提交方内联执行(提交点在响应写出后,不阻塞客户端),
显式配置 sample/drop 的部署不受影响
- 附带修复 ensure*Batcher 的存量数据竞争:channel nil 检查移入 sync.Once
正常负载零行为变化;突发时短暂等待替代永久丢失;数据库只见批量 INSERT。
This commit is contained in:
1 parent
28c2d20d9f
commit
a1b2b32e08
7 files changed
+92
-31
No files matched your search
@@ -372,12 +372,13 @@ func (r *usageLogRepository) CreateBestEffort(ctx context.Context, log *service.
|
||||
}
|
||||
}
|
||||
|
||||
// 队列满时阻塞等待而非立即丢弃:批处理器持续排空队列,短暂等待即可入队。
|
||||
// 立即丢弃会造成“已扣费但无 usage_log”的永久数据缺口(issue #3656);
|
||||
// 阻塞上限由调用方 ctx 期限约束,超时后由上层同步兜底。
|
||||
select {
|
||||
case r.bestEffortBatchCh <- req:
|
||||
case <-ctx.Done():
|
||||
return service.MarkUsageLogCreateDropped(ctx.Err())
|
||||
default:
|
||||
return service.MarkUsageLogCreateDropped(errors.New("usage log best-effort queue full"))
|
||||
}
|
||||
|
||||
select {
|
||||
@@ -493,12 +494,12 @@ func (r *usageLogRepository) createBatched(ctx context.Context, log *service.Usa
|
||||
resultCh: make(chan usageLogCreateResult, 1),
|
||||
}
|
||||
|
||||
// 队列满时阻塞等待而非立即报错:本路径是 best-effort 丢弃后的最后兜底,
|
||||
// 立即失败会让日志永久丢失;阻塞上限由调用方 ctx 期限约束。
|
||||
select {
|
||||
case r.createBatchCh <- req:
|
||||
case <-ctx.Done():
|
||||
return false, service.MarkUsageLogCreateNotPersisted(ctx.Err())
|
||||
default:
|
||||
return false, service.MarkUsageLogCreateNotPersisted(errors.New("usage log create batch queue full"))
|
||||
}
|
||||
|
||||
select {
|
||||
@@ -520,22 +521,28 @@ func (r *usageLogRepository) createBatched(ctx context.Context, log *service.Usa
|
||||
}
|
||||
|
||||
func (r *usageLogRepository) ensureCreateBatcher() {
|
||||
if r == nil || r.db == nil || r.createBatchCh != nil {
|
||||
if r == nil || r.db == nil {
|
||||
return
|
||||
}
|
||||
// nil 检查必须在 Once 内部:在外层做无同步快路径读会与 Once 内的写构成数据竞争。
|
||||
r.createBatchOnce.Do(func() {
|
||||
r.createBatchCh = make(chan usageLogCreateRequest, usageLogCreateBatchQueueCap)
|
||||
go r.runCreateBatcher(r.db)
|
||||
if r.createBatchCh == nil {
|
||||
r.createBatchCh = make(chan usageLogCreateRequest, usageLogCreateBatchQueueCap)
|
||||
go r.runCreateBatcher(r.db)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (r *usageLogRepository) ensureBestEffortBatcher() {
|
||||
if r == nil || r.db == nil || r.bestEffortBatchCh != nil {
|
||||
if r == nil || r.db == nil {
|
||||
return
|
||||
}
|
||||
// 同 ensureCreateBatcher:nil 检查放在 Once 内部以避免数据竞争。
|
||||
r.bestEffortBatchOnce.Do(func() {
|
||||
r.bestEffortBatchCh = make(chan usageLogBestEffortRequest, usageLogBestEffortBatchQueueCap)
|
||||
go r.runBestEffortBatcher(r.db)
|
||||
if r.bestEffortBatchCh == nil {
|
||||
r.bestEffortBatchCh = make(chan usageLogBestEffortRequest, usageLogBestEffortBatchQueueCap)
|
||||
go r.runBestEffortBatcher(r.db)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -288,21 +288,21 @@ func TestUsageLogRepositoryCreateBestEffort_BatchPathDuplicateRequestID(t *testi
|
||||
}, 3*time.Second, 20*time.Millisecond)
|
||||
}
|
||||
|
||||
func TestUsageLogRepositoryCreateBestEffort_QueueFullReturnsDropped(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
func TestUsageLogRepositoryCreateBestEffort_QueueFullBlocksUntilCtxDeadline(t *testing.T) {
|
||||
// 队列满时不再立即丢弃:阻塞等待入队,直到调用方 ctx 到期才标记 dropped(issue #3656)。
|
||||
client := testEntClient(t)
|
||||
repo := newUsageLogRepositoryWithSQL(client, integrationDB)
|
||||
repo.bestEffortBatchCh = make(chan usageLogBestEffortRequest, 1)
|
||||
repo.bestEffortBatchCh <- usageLogBestEffortRequest{}
|
||||
|
||||
user := mustCreateUser(t, client, &service.User{Email: fmt.Sprintf("usage-best-effort-full-%d@example.com", time.Now().UnixNano())})
|
||||
apiKey := mustCreateApiKey(t, client, &service.APIKey{UserID: user.ID, Key: "sk-usage-best-effort-full-" + uuid.NewString(), Name: "k"})
|
||||
account := mustCreateAccount(t, client, &service.Account{Name: "acc-usage-best-effort-full-" + uuid.NewString()})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
err := repo.CreateBestEffort(ctx, &service.UsageLog{
|
||||
UserID: user.ID,
|
||||
APIKeyID: apiKey.ID,
|
||||
AccountID: account.ID,
|
||||
UserID: 1,
|
||||
APIKeyID: 2,
|
||||
AccountID: 3,
|
||||
RequestID: uuid.NewString(),
|
||||
Model: "claude-3",
|
||||
InputTokens: 10,
|
||||
@@ -314,6 +314,40 @@ func TestUsageLogRepositoryCreateBestEffort_QueueFullReturnsDropped(t *testing.T
|
||||
|
||||
require.Error(t, err)
|
||||
require.True(t, service.IsUsageLogCreateDropped(err))
|
||||
require.GreaterOrEqual(t, time.Since(start), 150*time.Millisecond)
|
||||
}
|
||||
|
||||
func TestUsageLogRepositoryCreateBestEffort_QueueFullWaitsForDrain(t *testing.T) {
|
||||
// 队列满但批处理器随后排空时,阻塞的入队应成功完成而非丢弃。
|
||||
client := testEntClient(t)
|
||||
repo := newUsageLogRepositoryWithSQL(client, integrationDB)
|
||||
repo.bestEffortBatchCh = make(chan usageLogBestEffortRequest, 1)
|
||||
repo.bestEffortBatchCh <- usageLogBestEffortRequest{}
|
||||
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
<-repo.bestEffortBatchCh // 排空占位请求,为阻塞中的入队腾出空间
|
||||
req := <-repo.bestEffortBatchCh
|
||||
sendUsageLogBestEffortResult(req.resultCh, nil)
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := repo.CreateBestEffort(ctx, &service.UsageLog{
|
||||
UserID: 1,
|
||||
APIKeyID: 2,
|
||||
AccountID: 3,
|
||||
RequestID: uuid.NewString(),
|
||||
Model: "claude-3",
|
||||
InputTokens: 10,
|
||||
OutputTokens: 20,
|
||||
TotalCost: 0.5,
|
||||
ActualCost: 0.5,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestUsageLogRepositoryCreate_BatchPathCanceledContextMarksNotPersisted(t *testing.T) {
|
||||
@@ -346,7 +380,7 @@ func TestUsageLogRepositoryCreate_BatchPathCanceledContextMarksNotPersisted(t *t
|
||||
}
|
||||
|
||||
func TestUsageLogRepositoryCreate_BatchPathQueueFullMarksNotPersisted(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
// 队列满时阻塞等待入队,直到调用方 ctx 到期才标记 not persisted(issue #3656)。
|
||||
client := testEntClient(t)
|
||||
repo := newUsageLogRepositoryWithSQL(client, integrationDB)
|
||||
repo.createBatchCh = make(chan usageLogCreateRequest, 1)
|
||||
@@ -356,6 +390,10 @@ func TestUsageLogRepositoryCreate_BatchPathQueueFullMarksNotPersisted(t *testing
|
||||
apiKey := mustCreateApiKey(t, client, &service.APIKey{UserID: user.ID, Key: "sk-usage-create-full-" + uuid.NewString(), Name: "k"})
|
||||
account := mustCreateAccount(t, client, &service.Account{Name: "acc-usage-create-full-" + uuid.NewString()})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
inserted, err := repo.Create(ctx, &service.UsageLog{
|
||||
UserID: user.ID,
|
||||
APIKeyID: apiKey.ID,
|
||||
@@ -372,6 +410,7 @@ func TestUsageLogRepositoryCreate_BatchPathQueueFullMarksNotPersisted(t *testing
|
||||
require.False(t, inserted)
|
||||
require.Error(t, err)
|
||||
require.True(t, service.IsUsageLogCreateNotPersisted(err))
|
||||
require.GreaterOrEqual(t, time.Since(start), 150*time.Millisecond)
|
||||
}
|
||||
|
||||
func TestUsageLogRepositoryCreate_BatchPathCanceledAfterQueueMarksNotPersisted(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user