fix(batch-image): 修复审计发现的计费死锁、状态机与队列原子性缺陷

修复 PR #3768 批量图像 MVP 合并后审计报告中的全部问题:

结算与计费(高危):
- 所有 SETTLEMENT_* 失败(超冻结/计数非法/manifest 冲突/定价缺失/扣费失败)
  统一计入 retry_count 并在耗尽时释放冻结转 failed,消灭 settling 无限
  requeue 导致的冻结余额永久锁死
- 耗尽出口的释放指纹统一为 RequestHash,与 processor/Cancel/recovery 一致;
  release 遇同 request id 指纹冲突视为幂等成功,治愈历史毒消息
- 管理端校验 hold_multiplier >= discount_multiplier,定价快照对存量脏数据钳制
- 释放前校验 per-job hold claim(dedup+归档表),杜绝幻影释放

索引对账(高危):
- provider 输出与提交 custom_id 集对账:未知条目丢弃并记事件,
  漏项补 PROVIDER_RESULT_MISSING 失败行,保证 success+fail == item_count

提交与恢复(高危):
- 提交前转 uploading 并在 provider.Submit 期间心跳刷新 updated_at;
  恢复扫描改为原子复核(FailStaleUnsubmittedBatchImageJob),
  消灭慢提交被误杀退款而上游任务照常计费的孤儿场景
- 上游任务创建成功但本地状态推进失败时,尽力取消上游并清理输入
- recovery 释放失败时入队交由 worker releaseTerminalHold 兜底重试

队列与并发(中危):
- Enqueue(SetNX+LPush)与 Reserve(BRPop+ZAdd)均改为 Lua 原子脚本,
  消灭崩溃窗口导致 job 脱离队列、被 7 天 inflight 键锁死
- 锁冲突按 LockConflictDelay 重新入队(原直接丢弃需等 10 分钟 stale 恢复)
- 处理期间心跳:active zset 续期(ZAddXX 防幽灵成员)+ 锁 TTL 续期
- ReplaceBatchImageItemsForJob 增加 indexing 状态守卫,防掉队 worker 重写账目

存量回归(中危):
- image-only 定价条目(仅图片价无 token 价)恢复 token 计费 fail-closed,
  不再按 $0 计费;图片计费路径不受影响
- 鉴权余额门槛恢复 balance <= 0 语义,MinimumBalanceReserve 不再作硬 403

加固:
- ZIP max_items 钳制到管理员上限;Submit 补齐 Platform==Gemini 校验;
  gemini downloadUri 跟随前做 host 白名单校验
- 批量客户端改用共享 httpclient(拨号/TLS/响应头超时有界)
- 审计点名的忽略错误(MarkDownloaded/SettlementFailed/AppendEvent 等)改为记日志
This commit is contained in:
shaw committed 2026-07-07 18:53:56 +08:00
1 parent 44ab690a01
commit 80a229bce5
31 files changed
+1180 -90

No files matched your search

@@ -20,6 +20,10 @@ const (
defaultBatchImageLockPrefix = "batch_image:queue:lock:"
defaultBatchImageInflightTTL = 7 * 24 * time.Hour
defaultBatchImageJobLockTTL = 5 * time.Minute
// batchImageReservePollInterval 是原子 Reserve 脚本空轮询的间隔。
// 用轮询替代 BRPop 是为了保证 "弹出 + 写 active" 的原子性。
batchImageReservePollInterval = time.Second
)
var batchImageMoveDueDelayedScript = redis.NewScript(`
@@ -47,6 +51,36 @@ end
return 0
`)
var batchImageRefreshLockScript = redis.NewScript(`
if redis.call("GET", KEYS[1]) == ARGV[1] then
return redis.call("PEXPIRE", KEYS[1], ARGV[2])
end
return 0
`)
// batchImageReserveScript 原子地从 ready 弹出并写入 active zset。
// BRPop + ZAdd 两步方案在两步之间进程崩溃时 job 会脱离所有队列结构,
// 且 inflight 去重键(默认 7 天)会挡住所有重新入队。
var batchImageReserveScript = redis.NewScript(`
local job = redis.call("RPOP", KEYS[1])
if not job then
return nil
end
redis.call("ZADD", KEYS[2], ARGV[1], job)
return job
`)
// batchImageEnqueueScript 原子地设置 inflight 去重键并推入 ready。
// SetNX + LPush 两步方案在两步之间进程崩溃时,inflight 键(默认 7 天)
// 会挡住所有后续入队,而 job 从未进入 ready。
var batchImageEnqueueScript = redis.NewScript(`
if redis.call("SET", KEYS[1], ARGV[1], "NX", "PX", ARGV[2]) then
redis.call("LPUSH", KEYS[2], ARGV[1])
return 1
end
return 0
`)
type batchImageQueue struct {
rdb *redis.Client
readyKey string
@@ -131,40 +165,65 @@ func (q *batchImageQueue) Enqueue(ctx context.Context, batchID string) error {
return service.ErrInvalidBatchImageQueuePayload
}
ok, err := q.rdb.SetNX(ctx, q.inflightKey(batchID), batchID, q.inflightTTL).Result()
applied, err := batchImageEnqueueScript.Run(ctx, q.rdb,
[]string{q.inflightKey(batchID), q.readyKey},
batchID, q.inflightTTL.Milliseconds(),
).Int()
if err != nil {
return err
}
if !ok {
if applied == 0 {
return service.ErrBatchImageAlreadyQueued
}
if err := q.rdb.LPush(ctx, q.readyKey, batchID).Err(); err != nil {
_ = q.rdb.Del(ctx, q.inflightKey(batchID)).Err()
return err
}
return nil
}
func (q *batchImageQueue) Reserve(ctx context.Context, blockTimeout time.Duration) (service.ReservedBatchImageJob, error) {
result, err := q.rdb.BRPop(ctx, blockTimeout, q.readyKey).Result()
deadline := time.Now().Add(blockTimeout)
for {
batchID, err := q.reserveOnce(ctx)
if err == nil {
return service.ReservedBatchImageJob{BatchID: batchID}, nil
}
if !errors.Is(err, service.ErrBatchImageQueueEmpty) {
return service.ReservedBatchImageJob{}, err
}
remaining := time.Until(deadline)
if remaining <= 0 {
return service.ReservedBatchImageJob{}, service.ErrBatchImageQueueEmpty
}
wait := batchImageReservePollInterval
if remaining < wait {
wait = remaining
}
timer := time.NewTimer(wait)
select {
case <-ctx.Done():
timer.Stop()
return service.ReservedBatchImageJob{}, ctx.Err()
case <-timer.C:
}
}
}
func (q *batchImageQueue) reserveOnce(ctx context.Context) (string, error) {
raw, err := batchImageReserveScript.Run(ctx, q.rdb, []string{q.readyKey, q.activeKey}, time.Now().UnixMilli()).Result()
if errors.Is(err, redis.Nil) {
return service.ReservedBatchImageJob{}, service.ErrBatchImageQueueEmpty
return "", service.ErrBatchImageQueueEmpty
}
if err != nil {
return service.ReservedBatchImageJob{}, err
return "", err
}
if len(result) != 2 || !service.IsValidBatchImageID(result[1]) {
return service.ReservedBatchImageJob{}, service.ErrInvalidBatchImageQueuePayload
batchID, ok := raw.(string)
if !ok || !service.IsValidBatchImageID(batchID) {
// 非法 payload 已被脚本写入 active,必须移除,否则 stale 恢复会把它
// 无限重投回 ready。
if ok && batchID != "" {
_ = q.rdb.ZRem(ctx, q.activeKey, batchID).Err()
}
return "", service.ErrInvalidBatchImageQueuePayload
}
batchID := result[1]
if err := q.rdb.ZAdd(ctx, q.activeKey, redis.Z{
Score: float64(time.Now().UnixMilli()),
Member: batchID,
}).Err(); err != nil {
return service.ReservedBatchImageJob{}, err
}
return service.ReservedBatchImageJob{BatchID: batchID}, nil
return batchID, nil
}
func (q *batchImageQueue) RequeueAfter(ctx context.Context, batchID string, delay time.Duration) error {
@@ -202,7 +261,9 @@ func (q *batchImageQueue) Heartbeat(ctx context.Context, batchID string) error {
if !service.IsValidBatchImageID(batchID) {
return service.ErrInvalidBatchImageQueuePayload
}
return q.rdb.ZAdd(ctx, q.activeKey, redis.Z{
// XX:只刷新已存在的 active 成员。无条件 ZAdd 会在 Ack/Requeue 之后的
// 竞态心跳里把幽灵成员塞回 active zset。
return q.rdb.ZAddXX(ctx, q.activeKey, redis.Z{
Score: float64(time.Now().UnixMilli()),
Member: batchID,
}).Err()
@@ -269,6 +330,19 @@ func (l *batchImageRedisJobLock) Release(ctx context.Context) error {
return batchImageReleaseLockScript.Run(ctx, l.rdb, []string{l.key}, l.token).Err()
}
// Refresh 在仍持有锁(token 匹配)时续期 TTL,供长处理任务的心跳调用。
func (l *batchImageRedisJobLock) Refresh(ctx context.Context, ttl time.Duration) error {
if l == nil || l.rdb == nil || l.key == "" || l.token == "" {
return nil
}
if ttl <= 0 {
ttl = defaultBatchImageJobLockTTL
}
return batchImageRefreshLockScript.Run(ctx, l.rdb, []string{l.key}, l.token, ttl.Milliseconds()).Err()
}
var _ service.BatchImageJobLockRefresher = (*batchImageRedisJobLock)(nil)
func newBatchImageLockToken() (string, error) {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
@@ -108,6 +108,82 @@ func TestBatchImageQueue_JobLockReleaseOnlyDeletesMatchingToken(t *testing.T) {
require.ErrorIs(t, queue.rdb.Get(ctx, queue.lockKey(batchID)).Err(), redis.Nil)
}
func TestBatchImageQueue_ReserveAtomicallyMovesJobToActive(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
batchID := "imgbatch_reserve"
require.NoError(t, queue.Enqueue(ctx, batchID))
reserved, err := queue.Reserve(ctx, time.Second)
require.NoError(t, err)
require.Equal(t, batchID, reserved.BatchID)
// 弹出与写入 active 必须原子完成:ready 已空,active 中有该 job。
require.Equal(t, int64(0), queue.rdb.LLen(ctx, queue.readyKey).Val())
score, err := queue.rdb.ZScore(ctx, queue.activeKey, batchID).Result()
require.NoError(t, err)
require.Positive(t, score)
}
func TestBatchImageQueue_ReserveReturnsEmptyAfterTimeout(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
start := time.Now()
_, err := queue.Reserve(ctx, 50*time.Millisecond)
require.ErrorIs(t, err, service.ErrBatchImageQueueEmpty)
require.Less(t, time.Since(start), 5*time.Second)
}
func TestBatchImageQueue_ReserveDropsInvalidPayload(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
require.NoError(t, queue.rdb.LPush(ctx, queue.readyKey, "not-a-batch-id").Err())
_, err := queue.Reserve(ctx, 10*time.Millisecond)
require.ErrorIs(t, err, service.ErrInvalidBatchImageQueuePayload)
// 非法 payload 不得残留在 active zset,否则 stale 恢复会无限重投。
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, "not-a-batch-id").Err(), redis.Nil)
}
func TestBatchImageQueue_HeartbeatOnlyRefreshesExistingActiveMember(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
batchID := "imgbatch_heartbeat"
// 不在 active 中:心跳不得创建幽灵成员。
require.NoError(t, queue.Heartbeat(ctx, batchID))
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, batchID).Err(), redis.Nil)
require.NoError(t, queue.rdb.ZAdd(ctx, queue.activeKey, redis.Z{Score: 1, Member: batchID}).Err())
require.NoError(t, queue.Heartbeat(ctx, batchID))
score, err := queue.rdb.ZScore(ctx, queue.activeKey, batchID).Result()
require.NoError(t, err)
require.Greater(t, score, float64(1))
}
func TestBatchImageQueue_JobLockRefreshExtendsTTLOnlyForHolder(t *testing.T) {
ctx := context.Background()
queue, mr := newBatchImageQueueTest(t)
batchID := "imgbatch_lock_refresh"
lock, ok, err := queue.TryAcquireJobLock(ctx, batchID, time.Minute)
require.NoError(t, err)
require.True(t, ok)
refresher, isRefresher := lock.(service.BatchImageJobLockRefresher)
require.True(t, isRefresher)
require.NoError(t, refresher.Refresh(ctx, 10*time.Minute))
ttl := mr.TTL(queue.lockKey(batchID))
require.Greater(t, ttl, 5*time.Minute)
// token 不匹配时不得续期他人持有的锁。
require.NoError(t, queue.rdb.Set(ctx, queue.lockKey(batchID), "other-token", time.Minute).Err())
require.NoError(t, refresher.Refresh(ctx, 10*time.Minute))
ttl = mr.TTL(queue.lockKey(batchID))
require.LessOrEqual(t, ttl, time.Minute)
}
func newBatchImageQueueTest(t *testing.T) (*batchImageQueue, *miniredis.Miniredis) {
t.Helper()
mr := miniredis.RunT(t)
@@ -152,6 +152,45 @@ func (r *batchImageRepository) TransitionBatchImageJobStatus(ctx context.Context
return tx.Commit()
}
func (r *batchImageRepository) TouchBatchImageJobSubmitting(ctx context.Context, batchID string) error {
_, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET updated_at = $2
WHERE batch_id = $1
AND status IN ('created', 'uploading')`, batchID, time.Now())
return err
}
func (r *batchImageRepository) FailStaleUnsubmittedBatchImageJob(ctx context.Context, batchID string, cutoff time.Time, code, message string) (bool, error) {
now := time.Now()
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET status = 'failed',
last_error_code = $2,
last_error_message = $3,
finished_at = CASE WHEN finished_at IS NULL THEN $4 ELSE finished_at END,
updated_at = $4,
version = version + 1
WHERE batch_id = $1
AND status IN ('created', 'uploading')
AND provider_job_name IS NULL
AND updated_at <= $5`, batchID, code, message, now, cutoff)
if err != nil {
return false, err
}
affected, err := res.RowsAffected()
if err != nil {
return false, err
}
if affected == 0 {
return false, nil
}
return true, appendBatchImageEventWithSQL(ctx, r.sql, batchID, "billing_hold_recovery_failed_unsubmitted", map[string]any{
"batch_id": batchID,
"error_code": code,
})
}
func (r *batchImageRepository) UpdateBatchImageJobProviderOutputRef(ctx context.Context, batchID, providerOutputRef string) error {
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
@@ -410,9 +449,15 @@ func (r *batchImageRepository) ReplaceBatchImageItemsForJob(ctx context.Context,
func (r *batchImageRepository) replaceBatchImageItemsForJobWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID string, items []service.CreateBatchImageItemParams, counts service.BatchImageCounts) error {
var id int64
if err := sqlq.QueryRowContext(ctx, `SELECT id FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, batchID).Scan(&id); err != nil {
var status string
if err := sqlq.QueryRowContext(ctx, `SELECT id, status FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, batchID).Scan(&id, &status); err != nil {
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
// 仅允许 indexing 状态重建 item 表:防止锁过期后掉队的 worker
// 重写已完成/已结算 job 的条目,造成账目与结果漂移。
if status != service.BatchImageJobStatusIndexing {
return service.ErrBatchImageIndexStateConflict
}
promptPreviews, err := r.batchImageItemPromptPreviews(ctx, sqlq, batchID)
if err != nil {
return err
@@ -335,6 +335,16 @@ func releaseUsageBillingBatchImageBalance(ctx context.Context, tx *sql.Tx, cmd *
if cmd.HoldAmount <= 0 {
return &service.BatchImageBalanceHoldResult{}, nil
}
// 释放前校验该 job 确实预留过 hold(hold request id 已被 claim),
// 防止从未成功冻结的 job 触发"幻影释放",从其他用户的冻结资金池中凭空生成余额。
held, heldErr := batchImageHoldClaimExists(ctx, tx, service.BatchImageHoldRequestID(cmd.BatchID), cmd.APIKeyID)
if heldErr != nil {
return nil, heldErr
}
if !held {
logger.LegacyPrintf("repository.usage_billing", "[BatchImage] release skipped, hold was never reserved: batch=%s", cmd.BatchID)
return &service.BatchImageBalanceHoldResult{}, nil
}
var balance, frozen float64
err := tx.QueryRowContext(ctx, `
UPDATE users
@@ -358,6 +368,35 @@ func releaseUsageBillingBatchImageBalance(ctx context.Context, tx *sql.Tx, cmd *
return nil, errors.New("batch image frozen balance is insufficient")
}
// batchImageHoldClaimExists 检查 hold request id 是否已在 dedup(或归档)表中被 claim,
// 即该 batch 的冻结操作确实成功提交过。
func batchImageHoldClaimExists(ctx context.Context, tx *sql.Tx, holdRequestID string, apiKeyID int64) (bool, error) {
var exists int
err := tx.QueryRowContext(ctx, `
SELECT 1
FROM usage_billing_dedup
WHERE request_id = $1 AND api_key_id = $2
`, holdRequestID, apiKeyID).Scan(&exists)
if err == nil {
return true, nil
}
if !errors.Is(err, sql.ErrNoRows) {
return false, err
}
err = tx.QueryRowContext(ctx, `
SELECT 1
FROM usage_billing_dedup_archive
WHERE request_id = $1 AND api_key_id = $2
`, holdRequestID, apiKeyID).Scan(&exists)
if err == nil {
return true, nil
}
if errors.Is(err, sql.ErrNoRows) {
return false, nil
}
return false, err
}
func userExistsForBilling(ctx context.Context, tx *sql.Tx, userID int64) (bool, error) {
var exists int
err := tx.QueryRowContext(ctx, `
@@ -217,15 +217,45 @@ func TestReleaseUsageBillingBatchImageBalance_ReturnsFrozenToAvailable(t *testin
mock.ExpectBegin()
tx, err := db.BeginTx(ctx, nil)
require.NoError(t, err)
mock.ExpectQuery(`SELECT 1\s+FROM usage_billing_dedup\s+WHERE request_id = \$1 AND api_key_id = \$2`).
WithArgs(service.BatchImageHoldRequestID("imgbatch_release"), int64(7)).
WillReturnRows(sqlmock.NewRows([]string{"?column?"}).AddRow(1))
mock.ExpectQuery(releaseBatchImageHoldSQL).
WithArgs(1.0, int64(42)).
WillReturnRows(sqlmock.NewRows([]string{"balance", "frozen_balance"}).AddRow(10.0, 0.0))
mock.ExpectCommit()
result, err := releaseUsageBillingBatchImageBalance(ctx, tx, &service.BatchImageBalanceHoldCommand{UserID: 42, HoldAmount: 1})
result, err := releaseUsageBillingBatchImageBalance(ctx, tx, &service.BatchImageBalanceHoldCommand{UserID: 42, APIKeyID: 7, BatchID: "imgbatch_release", HoldAmount: 1})
require.NoError(t, err)
require.InDelta(t, 10.0, *result.NewBalance, 0.000001)
require.InDelta(t, 0.0, *result.FrozenBalance, 0.000001)
require.NoError(t, tx.Commit())
require.NoError(t, mock.ExpectationsWereMet())
}
func TestReleaseUsageBillingBatchImageBalance_SkipsWhenHoldNeverReserved(t *testing.T) {
ctx := context.Background()
db, mock, err := sqlmock.New()
require.NoError(t, err)
defer func() { _ = db.Close() }()
mock.ExpectBegin()
tx, err := db.BeginTx(ctx, nil)
require.NoError(t, err)
// dedup 与归档表均无 hold claim:说明该 job 从未成功冻结,
// 释放必须跳过,不得从他人冻结资金池中凭空生成余额。
mock.ExpectQuery(`SELECT 1\s+FROM usage_billing_dedup\s+WHERE request_id = \$1 AND api_key_id = \$2`).
WithArgs(service.BatchImageHoldRequestID("imgbatch_phantom"), int64(7)).
WillReturnError(sql.ErrNoRows)
mock.ExpectQuery(`SELECT 1\s+FROM usage_billing_dedup_archive\s+WHERE request_id = \$1 AND api_key_id = \$2`).
WithArgs(service.BatchImageHoldRequestID("imgbatch_phantom"), int64(7)).
WillReturnError(sql.ErrNoRows)
mock.ExpectCommit()
result, err := releaseUsageBillingBatchImageBalance(ctx, tx, &service.BatchImageBalanceHoldCommand{UserID: 42, APIKeyID: 7, BatchID: "imgbatch_phantom", HoldAmount: 1})
require.NoError(t, err)
require.Nil(t, result.NewBalance)
require.Nil(t, result.FrozenBalance)
require.NoError(t, tx.Commit())
require.NoError(t, mock.ExpectationsWereMet())
}