mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-11-17 14:57:27 +00:00
feat: add batch image MVP
This commit is contained in:
1 parent
28c2d20d9f
commit
a994fbd77a
79 files changed
+34113
-36
No files matched your search
@@ -0,0 +1,112 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultBatchImageDownloadActivePrefix = "batch_image:download:active:"
|
||||
defaultBatchImageDownloadActiveTTL = 10 * time.Minute
|
||||
defaultBatchImageDownloadConcurrency = 2
|
||||
)
|
||||
|
||||
var batchImageDownloadAcquireScript = redis.NewScript(`
|
||||
local current = tonumber(redis.call("GET", KEYS[1]) or "0")
|
||||
local max = tonumber(ARGV[1])
|
||||
if current >= max then
|
||||
return 0
|
||||
end
|
||||
redis.call("INCR", KEYS[1])
|
||||
redis.call("EXPIRE", KEYS[1], ARGV[2])
|
||||
return 1
|
||||
`)
|
||||
|
||||
var batchImageDownloadReleaseScript = redis.NewScript(`
|
||||
local current = tonumber(redis.call("GET", KEYS[1]) or "0")
|
||||
if current <= 1 then
|
||||
redis.call("DEL", KEYS[1])
|
||||
return 0
|
||||
end
|
||||
return redis.call("DECR", KEYS[1])
|
||||
`)
|
||||
|
||||
type batchImageDownloadLimiter struct {
|
||||
rdb *redis.Client
|
||||
activePrefix string
|
||||
maxActive int
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
func NewBatchImageDownloadLimiter(rdb *redis.Client, cfg *config.Config) service.BatchImageDownloadLimiter {
|
||||
maxActive := defaultBatchImageDownloadConcurrency
|
||||
ttl := defaultBatchImageDownloadActiveTTL
|
||||
if cfg != nil {
|
||||
if cfg.BatchImage.MaxDownloadConcurrencyPerUser > 0 {
|
||||
maxActive = cfg.BatchImage.MaxDownloadConcurrencyPerUser
|
||||
}
|
||||
if cfg.BatchImage.MaxDownloadDurationSeconds > 0 {
|
||||
ttl = time.Duration(cfg.BatchImage.MaxDownloadDurationSeconds) * time.Second
|
||||
}
|
||||
}
|
||||
return &batchImageDownloadLimiter{
|
||||
rdb: rdb,
|
||||
activePrefix: defaultBatchImageDownloadActivePrefix,
|
||||
maxActive: maxActive,
|
||||
ttl: ttl,
|
||||
}
|
||||
}
|
||||
|
||||
func newBatchImageDownloadLimiterForTest(rdb *redis.Client, maxActive int, ttl time.Duration) *batchImageDownloadLimiter {
|
||||
if maxActive <= 0 {
|
||||
maxActive = defaultBatchImageDownloadConcurrency
|
||||
}
|
||||
if ttl <= 0 {
|
||||
ttl = defaultBatchImageDownloadActiveTTL
|
||||
}
|
||||
return &batchImageDownloadLimiter{rdb: rdb, activePrefix: defaultBatchImageDownloadActivePrefix, maxActive: maxActive, ttl: ttl}
|
||||
}
|
||||
|
||||
func (l *batchImageDownloadLimiter) Acquire(ctx context.Context, userID string, kind string) (service.BatchImageDownloadPermit, error) {
|
||||
if l == nil || l.rdb == nil {
|
||||
return nil, service.ErrBatchImageDownloadLimited
|
||||
}
|
||||
key := l.activeKey(userID)
|
||||
ok, err := batchImageDownloadAcquireScript.Run(ctx, l.rdb, []string{key}, l.maxActive, int(l.ttl.Seconds())).Int()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ok != 1 {
|
||||
return nil, service.ErrBatchImageDownloadLimited
|
||||
}
|
||||
return &batchImageDownloadPermit{rdb: l.rdb, key: key}, nil
|
||||
}
|
||||
|
||||
func (l *batchImageDownloadLimiter) activeKey(userID string) string {
|
||||
return l.activePrefix + userID
|
||||
}
|
||||
|
||||
type batchImageDownloadPermit struct {
|
||||
rdb *redis.Client
|
||||
key string
|
||||
once sync.Once
|
||||
err error
|
||||
}
|
||||
|
||||
func (p *batchImageDownloadPermit) Release(ctx context.Context) error {
|
||||
if p == nil || p.rdb == nil || p.key == "" {
|
||||
return nil
|
||||
}
|
||||
p.once.Do(func() {
|
||||
_, p.err = batchImageDownloadReleaseScript.Run(ctx, p.rdb, []string{p.key}).Result()
|
||||
})
|
||||
return p.err
|
||||
}
|
||||
|
||||
var _ service.BatchImageDownloadLimiter = (*batchImageDownloadLimiter)(nil)
|
||||
var _ service.BatchImageDownloadPermit = (*batchImageDownloadPermit)(nil)
|
||||
@@ -0,0 +1,38 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBatchImageDownloadLimiter_AcquireDenyReleaseAndTTL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
limiter := newBatchImageDownloadLimiterForTest(rdb, 1, time.Minute)
|
||||
|
||||
permit, err := limiter.Acquire(ctx, "11", "zip")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, permit)
|
||||
require.True(t, mr.TTL(limiter.activeKey("11")) > 0)
|
||||
|
||||
_, err = limiter.Acquire(ctx, "11", "zip")
|
||||
require.ErrorIs(t, err, service.ErrBatchImageDownloadLimited)
|
||||
|
||||
require.NoError(t, permit.Release(ctx))
|
||||
require.NoError(t, permit.Release(ctx))
|
||||
require.False(t, mr.Exists(limiter.activeKey("11")))
|
||||
|
||||
permit, err = limiter.Acquire(ctx, "11", "zip")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, permit)
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultBatchImageReadyKey = "batch_image:queue:ready"
|
||||
defaultBatchImageDelayedKey = "batch_image:queue:delayed"
|
||||
defaultBatchImageActiveKey = "batch_image:queue:active"
|
||||
defaultBatchImageInflightPrefix = "batch_image:queue:inflight:"
|
||||
defaultBatchImageLockPrefix = "batch_image:queue:lock:"
|
||||
defaultBatchImageInflightTTL = 7 * 24 * time.Hour
|
||||
defaultBatchImageJobLockTTL = 5 * time.Minute
|
||||
)
|
||||
|
||||
var batchImageMoveDueDelayedScript = redis.NewScript(`
|
||||
local jobs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, ARGV[2])
|
||||
for _, job in ipairs(jobs) do
|
||||
redis.call("ZREM", KEYS[1], job)
|
||||
redis.call("LPUSH", KEYS[2], job)
|
||||
end
|
||||
return #jobs
|
||||
`)
|
||||
|
||||
var batchImageRecoverStaleActiveScript = redis.NewScript(`
|
||||
local jobs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, ARGV[2])
|
||||
for _, job in ipairs(jobs) do
|
||||
redis.call("ZREM", KEYS[1], job)
|
||||
redis.call("LPUSH", KEYS[2], job)
|
||||
end
|
||||
return #jobs
|
||||
`)
|
||||
|
||||
var batchImageReleaseLockScript = redis.NewScript(`
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("DEL", KEYS[1])
|
||||
end
|
||||
return 0
|
||||
`)
|
||||
|
||||
type batchImageQueue struct {
|
||||
rdb *redis.Client
|
||||
readyKey string
|
||||
delayedKey string
|
||||
activeKey string
|
||||
inflightPrefix string
|
||||
lockPrefix string
|
||||
inflightTTL time.Duration
|
||||
lockTTL time.Duration
|
||||
}
|
||||
|
||||
func NewBatchImageQueue(rdb *redis.Client, cfg *config.Config) service.BatchImageQueue {
|
||||
return newBatchImageQueueWithOptions(rdb, batchImageQueueOptionsFromConfig(cfg))
|
||||
}
|
||||
|
||||
type batchImageQueueOptions struct {
|
||||
ReadyKey string
|
||||
DelayedKey string
|
||||
ActiveKey string
|
||||
InflightPrefix string
|
||||
LockPrefix string
|
||||
InflightTTL time.Duration
|
||||
LockTTL time.Duration
|
||||
}
|
||||
|
||||
func newBatchImageQueueWithOptions(rdb *redis.Client, opts batchImageQueueOptions) *batchImageQueue {
|
||||
opts = normalizeBatchImageQueueOptions(opts)
|
||||
return &batchImageQueue{
|
||||
rdb: rdb,
|
||||
readyKey: opts.ReadyKey,
|
||||
delayedKey: opts.DelayedKey,
|
||||
activeKey: opts.ActiveKey,
|
||||
inflightPrefix: opts.InflightPrefix,
|
||||
lockPrefix: opts.LockPrefix,
|
||||
inflightTTL: opts.InflightTTL,
|
||||
lockTTL: opts.LockTTL,
|
||||
}
|
||||
}
|
||||
|
||||
func batchImageQueueOptionsFromConfig(cfg *config.Config) batchImageQueueOptions {
|
||||
if cfg == nil {
|
||||
return batchImageQueueOptions{}
|
||||
}
|
||||
return batchImageQueueOptions{
|
||||
ReadyKey: cfg.BatchImage.QueueReadyKey,
|
||||
DelayedKey: cfg.BatchImage.QueueDelayedKey,
|
||||
ActiveKey: cfg.BatchImage.QueueActiveKey,
|
||||
InflightPrefix: cfg.BatchImage.InflightKeyPrefix,
|
||||
LockPrefix: cfg.BatchImage.LockKeyPrefix,
|
||||
InflightTTL: time.Duration(cfg.BatchImage.InflightTTLSeconds) * time.Second,
|
||||
LockTTL: time.Duration(cfg.BatchImage.JobLockTTLSeconds) * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeBatchImageQueueOptions(opts batchImageQueueOptions) batchImageQueueOptions {
|
||||
if opts.ReadyKey == "" {
|
||||
opts.ReadyKey = defaultBatchImageReadyKey
|
||||
}
|
||||
if opts.DelayedKey == "" {
|
||||
opts.DelayedKey = defaultBatchImageDelayedKey
|
||||
}
|
||||
if opts.ActiveKey == "" {
|
||||
opts.ActiveKey = defaultBatchImageActiveKey
|
||||
}
|
||||
if opts.InflightPrefix == "" {
|
||||
opts.InflightPrefix = defaultBatchImageInflightPrefix
|
||||
}
|
||||
if opts.LockPrefix == "" {
|
||||
opts.LockPrefix = defaultBatchImageLockPrefix
|
||||
}
|
||||
if opts.InflightTTL <= 0 {
|
||||
opts.InflightTTL = defaultBatchImageInflightTTL
|
||||
}
|
||||
if opts.LockTTL <= 0 {
|
||||
opts.LockTTL = defaultBatchImageJobLockTTL
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) Enqueue(ctx context.Context, batchID string) error {
|
||||
if !service.IsValidBatchImageID(batchID) {
|
||||
return service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
|
||||
ok, err := q.rdb.SetNX(ctx, q.inflightKey(batchID), batchID, q.inflightTTL).Result()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return service.ErrBatchImageAlreadyQueued
|
||||
}
|
||||
if err := q.rdb.LPush(ctx, q.readyKey, batchID).Err(); err != nil {
|
||||
_ = q.rdb.Del(ctx, q.inflightKey(batchID)).Err()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) Reserve(ctx context.Context, blockTimeout time.Duration) (service.ReservedBatchImageJob, error) {
|
||||
result, err := q.rdb.BRPop(ctx, blockTimeout, q.readyKey).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return service.ReservedBatchImageJob{}, service.ErrBatchImageQueueEmpty
|
||||
}
|
||||
if err != nil {
|
||||
return service.ReservedBatchImageJob{}, err
|
||||
}
|
||||
if len(result) != 2 || !service.IsValidBatchImageID(result[1]) {
|
||||
return service.ReservedBatchImageJob{}, service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
|
||||
batchID := result[1]
|
||||
if err := q.rdb.ZAdd(ctx, q.activeKey, redis.Z{
|
||||
Score: float64(time.Now().UnixMilli()),
|
||||
Member: batchID,
|
||||
}).Err(); err != nil {
|
||||
return service.ReservedBatchImageJob{}, err
|
||||
}
|
||||
return service.ReservedBatchImageJob{BatchID: batchID}, nil
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) RequeueAfter(ctx context.Context, batchID string, delay time.Duration) error {
|
||||
if !service.IsValidBatchImageID(batchID) {
|
||||
return service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
pipe := q.rdb.TxPipeline()
|
||||
pipe.ZRem(ctx, q.activeKey, batchID)
|
||||
pipe.ZRem(ctx, q.delayedKey, batchID)
|
||||
if delay <= 0 {
|
||||
pipe.LPush(ctx, q.readyKey, batchID)
|
||||
} else {
|
||||
pipe.ZAdd(ctx, q.delayedKey, redis.Z{
|
||||
Score: float64(time.Now().Add(delay).UnixMilli()),
|
||||
Member: batchID,
|
||||
})
|
||||
}
|
||||
_, err := pipe.Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) Ack(ctx context.Context, batchID string) error {
|
||||
if !service.IsValidBatchImageID(batchID) {
|
||||
return service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
pipe := q.rdb.TxPipeline()
|
||||
pipe.ZRem(ctx, q.activeKey, batchID)
|
||||
pipe.ZRem(ctx, q.delayedKey, batchID)
|
||||
pipe.Del(ctx, q.inflightKey(batchID))
|
||||
_, err := pipe.Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) Heartbeat(ctx context.Context, batchID string) error {
|
||||
if !service.IsValidBatchImageID(batchID) {
|
||||
return service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
return q.rdb.ZAdd(ctx, q.activeKey, redis.Z{
|
||||
Score: float64(time.Now().UnixMilli()),
|
||||
Member: batchID,
|
||||
}).Err()
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) MoveDueDelayedToReady(ctx context.Context, limit int) (int, error) {
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
return batchImageMoveDueDelayedScript.Run(ctx, q.rdb, []string{q.delayedKey, q.readyKey}, time.Now().UnixMilli(), limit).Int()
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) RecoverStaleActive(ctx context.Context, staleAfter time.Duration, limit int) (int, error) {
|
||||
if staleAfter <= 0 {
|
||||
return 0, service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
cutoff := time.Now().Add(-staleAfter).UnixMilli()
|
||||
return batchImageRecoverStaleActiveScript.Run(ctx, q.rdb, []string{q.activeKey, q.readyKey}, cutoff, limit).Int()
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) TryAcquireJobLock(ctx context.Context, batchID string, ttl time.Duration) (service.BatchImageJobLock, bool, error) {
|
||||
if !service.IsValidBatchImageID(batchID) {
|
||||
return nil, false, service.ErrInvalidBatchImageQueuePayload
|
||||
}
|
||||
if ttl <= 0 {
|
||||
ttl = q.lockTTL
|
||||
}
|
||||
token, err := newBatchImageLockToken()
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
key := q.lockKey(batchID)
|
||||
ok, err := q.rdb.SetNX(ctx, key, token, ttl).Result()
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !ok {
|
||||
return nil, false, nil
|
||||
}
|
||||
return &batchImageRedisJobLock{rdb: q.rdb, key: key, token: token}, true, nil
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) inflightKey(batchID string) string {
|
||||
return q.inflightPrefix + batchID
|
||||
}
|
||||
|
||||
func (q *batchImageQueue) lockKey(batchID string) string {
|
||||
return q.lockPrefix + batchID
|
||||
}
|
||||
|
||||
type batchImageRedisJobLock struct {
|
||||
rdb *redis.Client
|
||||
key string
|
||||
token string
|
||||
}
|
||||
|
||||
func (l *batchImageRedisJobLock) Release(ctx context.Context) error {
|
||||
if l == nil || l.rdb == nil || l.key == "" || l.token == "" {
|
||||
return nil
|
||||
}
|
||||
return batchImageReleaseLockScript.Run(ctx, l.rdb, []string{l.key}, l.token).Err()
|
||||
}
|
||||
|
||||
func newBatchImageLockToken() (string, error) {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(b[:]), nil
|
||||
}
|
||||
|
||||
var _ service.BatchImageQueue = (*batchImageQueue)(nil)
|
||||
@@ -0,0 +1,123 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBatchImageQueue_DuplicateEnqueueReturnsAlreadyQueued(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
batchID := "imgbatch_duplicate"
|
||||
|
||||
require.NoError(t, queue.Enqueue(ctx, batchID))
|
||||
err := queue.Enqueue(ctx, batchID)
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, service.ErrBatchImageAlreadyQueued))
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_RequeueAfterMovesJobFromActiveToDelayed(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
batchID := "imgbatch_requeue_after"
|
||||
require.NoError(t, queue.rdb.ZAdd(ctx, queue.activeKey, redis.Z{
|
||||
Score: float64(time.Now().UnixMilli()),
|
||||
Member: batchID,
|
||||
}).Err())
|
||||
|
||||
require.NoError(t, queue.RequeueAfter(ctx, batchID, time.Minute))
|
||||
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, batchID).Err(), redis.Nil)
|
||||
score, err := queue.rdb.ZScore(ctx, queue.delayedKey, batchID).Result()
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, score, float64(time.Now().UnixMilli()))
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_MoveDueDelayedToReadyMovesDueJobs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
dueBatchID := "imgbatch_due"
|
||||
futureBatchID := "imgbatch_future"
|
||||
now := time.Now()
|
||||
require.NoError(t, queue.rdb.ZAdd(ctx, queue.delayedKey,
|
||||
redis.Z{Score: float64(now.Add(-time.Second).UnixMilli()), Member: dueBatchID},
|
||||
redis.Z{Score: float64(now.Add(time.Hour).UnixMilli()), Member: futureBatchID},
|
||||
).Err())
|
||||
|
||||
moved, err := queue.MoveDueDelayedToReady(ctx, 10)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, moved)
|
||||
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.delayedKey, dueBatchID).Err(), redis.Nil)
|
||||
require.NoError(t, queue.rdb.ZScore(ctx, queue.delayedKey, futureBatchID).Err())
|
||||
|
||||
reserved, err := queue.Reserve(ctx, time.Millisecond)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, dueBatchID, reserved.BatchID)
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_RecoverStaleActiveMovesStaleJobsToReady(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
staleBatchID := "imgbatch_stale"
|
||||
recentBatchID := "imgbatch_recent"
|
||||
now := time.Now()
|
||||
require.NoError(t, queue.rdb.ZAdd(ctx, queue.activeKey,
|
||||
redis.Z{Score: float64(now.Add(-time.Hour).UnixMilli()), Member: staleBatchID},
|
||||
redis.Z{Score: float64(now.UnixMilli()), Member: recentBatchID},
|
||||
).Err())
|
||||
|
||||
moved, err := queue.RecoverStaleActive(ctx, 10*time.Minute, 10)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, moved)
|
||||
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, staleBatchID).Err(), redis.Nil)
|
||||
require.NoError(t, queue.rdb.ZScore(ctx, queue.activeKey, recentBatchID).Err())
|
||||
|
||||
reserved, err := queue.Reserve(ctx, time.Millisecond)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, staleBatchID, reserved.BatchID)
|
||||
}
|
||||
|
||||
func TestBatchImageQueue_JobLockReleaseOnlyDeletesMatchingToken(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
queue, _ := newBatchImageQueueTest(t)
|
||||
batchID := "imgbatch_lock"
|
||||
|
||||
lock, ok, err := queue.TryAcquireJobLock(ctx, batchID, time.Minute)
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
|
||||
require.NoError(t, queue.rdb.Set(ctx, queue.lockKey(batchID), "other-token", time.Minute).Err())
|
||||
require.NoError(t, lock.Release(ctx))
|
||||
got, err := queue.rdb.Get(ctx, queue.lockKey(batchID)).Result()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "other-token", got)
|
||||
|
||||
require.NoError(t, queue.rdb.Del(ctx, queue.lockKey(batchID)).Err())
|
||||
lock, ok, err = queue.TryAcquireJobLock(ctx, batchID, time.Minute)
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
require.NoError(t, lock.Release(ctx))
|
||||
require.ErrorIs(t, queue.rdb.Get(ctx, queue.lockKey(batchID)).Err(), redis.Nil)
|
||||
}
|
||||
|
||||
func newBatchImageQueueTest(t *testing.T) (*batchImageQueue, *miniredis.Miniredis) {
|
||||
t.Helper()
|
||||
mr := miniredis.RunT(t)
|
||||
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||
t.Cleanup(func() {
|
||||
_ = rdb.Close()
|
||||
})
|
||||
queue := newBatchImageQueueWithOptions(rdb, batchImageQueueOptions{
|
||||
InflightTTL: time.Hour,
|
||||
LockTTL: time.Minute,
|
||||
})
|
||||
return queue, mr
|
||||
}
|
||||
@@ -0,0 +1,782 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
|
||||
type batchImageSQLExecutor interface {
|
||||
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
|
||||
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
|
||||
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
||||
}
|
||||
|
||||
type batchImageRepository struct {
|
||||
db *sql.DB
|
||||
sql batchImageSQLExecutor
|
||||
}
|
||||
|
||||
func NewBatchImageRepository(db *sql.DB) service.BatchImageRepository {
|
||||
return &batchImageRepository{db: db, sql: db}
|
||||
}
|
||||
|
||||
func newBatchImageRepositoryWithSQL(sqlq batchImageSQLExecutor) *batchImageRepository {
|
||||
return &batchImageRepository{sql: sqlq}
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) CreateBatchImageJob(ctx context.Context, params service.CreateBatchImageJobParams) (*service.BatchImageJob, error) {
|
||||
if !service.IsSupportedBatchImageProvider(params.Provider) {
|
||||
return nil, service.ErrBatchImageInvalidProvider
|
||||
}
|
||||
if params.BatchID == "" {
|
||||
batchID, err := service.NewBatchImageID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
params.BatchID = batchID
|
||||
}
|
||||
if params.Status == "" {
|
||||
params.Status = service.BatchImageJobStatusCreated
|
||||
}
|
||||
if params.Currency == "" {
|
||||
params.Currency = "USD"
|
||||
}
|
||||
|
||||
job, err := createBatchImageJobWithSQL(ctx, r.sql, params)
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, nil, service.ErrBatchImageJobExists)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) GetBatchImageJobByBatchID(ctx context.Context, batchID string) (*service.BatchImageJob, error) {
|
||||
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+" WHERE batch_id = $1", batchID))
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) GetBatchImageJobByIdempotencyKey(ctx context.Context, userID, apiKeyID int64, key string) (*service.BatchImageJob, error) {
|
||||
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+`
|
||||
WHERE user_id = $1 AND api_key_id = $2 AND idempotency_key = $3
|
||||
ORDER BY id DESC LIMIT 1`, userID, apiKeyID, key))
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) GetBatchImageJobByBatchIDForOwner(ctx context.Context, userID, apiKeyID int64, batchID string) (*service.BatchImageJob, error) {
|
||||
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+`
|
||||
WHERE batch_id = $1 AND user_id = $2 AND api_key_id = $3`, batchID, userID, apiKeyID))
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) GetBatchImageJobByID(ctx context.Context, id int64) (*service.BatchImageJob, error) {
|
||||
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+" WHERE id = $1", id))
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) TransitionBatchImageJobStatus(ctx context.Context, batchID, toStatus string, opts service.BatchImageTransitionOptions) error {
|
||||
if r.db == nil {
|
||||
return r.transitionBatchImageJobStatusWithSQL(ctx, r.sql, batchID, toStatus, opts)
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
if err := r.transitionBatchImageJobStatusWithSQL(ctx, tx, batchID, toStatus, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) UpdateBatchImageJobProviderOutputRef(ctx context.Context, batchID, providerOutputRef string) error {
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET provider_output_ref = $2, updated_at = $3
|
||||
WHERE batch_id = $1`, batchID, providerOutputRef, time.Now())
|
||||
if err != nil {
|
||||
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
|
||||
return service.ErrBatchImageJobNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) UpdateBatchImageJobProviderSubmit(ctx context.Context, params service.UpdateBatchImageJobProviderSubmitParams) error {
|
||||
if r.db == nil {
|
||||
return r.updateBatchImageJobProviderSubmitWithSQL(ctx, r.sql, params)
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
if err := r.updateBatchImageJobProviderSubmitWithSQL(ctx, tx, params); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) updateBatchImageJobProviderSubmitWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.UpdateBatchImageJobProviderSubmitParams) error {
|
||||
var current string
|
||||
if err := sqlq.QueryRowContext(ctx, `SELECT status FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, params.BatchID).Scan(¤t); err != nil {
|
||||
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
if !service.CanTransitionBatchImageJob(current, service.BatchImageJobStatusSubmitted) {
|
||||
return service.ErrBatchImageInvalidTransition
|
||||
}
|
||||
now := time.Now()
|
||||
if _, err := sqlq.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET status = 'submitted',
|
||||
provider_job_name = $2,
|
||||
provider_input_ref = NULLIF($3, ''),
|
||||
provider_output_ref = NULLIF($4, ''),
|
||||
gcs_input_uri = NULLIF($5, ''),
|
||||
gcs_output_uri = NULLIF($6, ''),
|
||||
submitted_at = CASE WHEN submitted_at IS NULL THEN $7 ELSE submitted_at END,
|
||||
updated_at = $7,
|
||||
version = version + 1
|
||||
WHERE batch_id = $1`, params.BatchID, params.ProviderJobName, params.ProviderInputRef, params.ProviderOutputRef, params.GCSInputURI, params.GCSOutputURI, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, sqlq, params.BatchID, "provider_submitted", params.EventPayload)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) RecordBatchImageJobSubmitFailure(ctx context.Context, batchID, code, message string, markFailed bool) error {
|
||||
now := time.Now()
|
||||
statusSQL := "status"
|
||||
if markFailed {
|
||||
statusSQL = "'failed'"
|
||||
}
|
||||
_, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET status = `+statusSQL+`,
|
||||
last_error_code = $2,
|
||||
last_error_message = $3,
|
||||
finished_at = CASE WHEN `+statusSQL+` = 'failed' AND finished_at IS NULL THEN $4 ELSE finished_at END,
|
||||
updated_at = $4,
|
||||
version = version + 1
|
||||
WHERE batch_id = $1`, batchID, code, message, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
eventType := "submit_failed"
|
||||
if !markFailed {
|
||||
eventType = "queue_failed"
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, eventType, map[string]any{"error_code": code})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) MarkBatchImageJobSettled(ctx context.Context, params service.MarkBatchImageJobSettledParams) error {
|
||||
if r.db == nil {
|
||||
return r.markBatchImageJobSettledWithSQL(ctx, r.sql, params)
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
if err := r.markBatchImageJobSettledWithSQL(ctx, tx, params); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) markBatchImageJobSettledWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.MarkBatchImageJobSettledParams) error {
|
||||
now := time.Now()
|
||||
if params.Now != nil {
|
||||
now = *params.Now
|
||||
}
|
||||
outputExpiresAt := params.OutputExpiresAt
|
||||
|
||||
res, err := sqlq.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET status = 'completed',
|
||||
actual_cost = $2,
|
||||
manifest_hash = $3,
|
||||
settled_at = CASE WHEN settled_at IS NULL THEN $4 ELSE settled_at END,
|
||||
finished_at = CASE WHEN finished_at IS NULL THEN $4 ELSE finished_at END,
|
||||
output_expires_at = CASE WHEN output_expires_at IS NULL THEN $5 ELSE output_expires_at END,
|
||||
updated_at = $4,
|
||||
version = version + 1
|
||||
WHERE batch_id = $1
|
||||
AND status = 'settling'
|
||||
AND (manifest_hash IS NULL OR manifest_hash = '' OR manifest_hash = $3)`, params.BatchID, params.ActualCost, params.ManifestHash, now, outputExpiresAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected == 0 {
|
||||
job, getErr := scanBatchImageJob(sqlq.QueryRowContext(ctx, batchImageJobSelectSQL+" WHERE batch_id = $1", params.BatchID))
|
||||
if getErr != nil {
|
||||
return translatePersistenceError(getErr, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
if job.Status != service.BatchImageJobStatusSettling {
|
||||
if job.Status == service.BatchImageJobStatusCompleted {
|
||||
return service.ErrBatchImageAlreadySettled
|
||||
}
|
||||
return service.ErrBatchImageSettlementInvalidStatus
|
||||
}
|
||||
return service.ErrBatchImageSettlementManifestConflict
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, sqlq, params.BatchID, "settlement_completed", params.EventPayload)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) SetBatchImageJobSettlementFailed(ctx context.Context, batchID, code, message string) error {
|
||||
_, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET last_error_code = $2,
|
||||
last_error_message = $3,
|
||||
retry_count = retry_count + 1,
|
||||
updated_at = $4
|
||||
WHERE batch_id = $1`, batchID, code, message, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "settlement_failed", map[string]any{
|
||||
"error_code": code,
|
||||
})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) transitionBatchImageJobStatusWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID, toStatus string, opts service.BatchImageTransitionOptions) error {
|
||||
var current string
|
||||
if err := sqlq.QueryRowContext(ctx, `SELECT status FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, batchID).Scan(¤t); err != nil {
|
||||
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
if !service.CanTransitionBatchImageJob(current, toStatus) {
|
||||
return service.ErrBatchImageInvalidTransition
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if opts.Now != nil {
|
||||
now = *opts.Now
|
||||
}
|
||||
|
||||
if _, err := sqlq.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET
|
||||
status = $2,
|
||||
version = version + 1,
|
||||
updated_at = $3,
|
||||
last_error_code = CASE WHEN $2 = 'failed' THEN $4 ELSE last_error_code END,
|
||||
last_error_message = CASE WHEN $2 = 'failed' THEN $5 ELSE last_error_message END,
|
||||
submitted_at = CASE WHEN $2 = 'submitted' AND submitted_at IS NULL THEN $3 ELSE submitted_at END,
|
||||
started_at = CASE WHEN $2 = 'running' AND started_at IS NULL THEN $3 ELSE started_at END,
|
||||
finished_at = CASE WHEN $2 IN ('completed', 'failed', 'cancelled') AND finished_at IS NULL THEN $3 ELSE finished_at END,
|
||||
settled_at = CASE WHEN $2 = 'completed' AND settled_at IS NULL THEN $3 ELSE settled_at END,
|
||||
output_deleted_at = CASE WHEN $2 = 'output_deleted' AND output_deleted_at IS NULL THEN $3 ELSE output_deleted_at END
|
||||
WHERE batch_id = $1`, batchID, toStatus, now, opts.ErrorCode, opts.ErrorMessage); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if opts.EventType != "" {
|
||||
return appendBatchImageEventWithSQL(ctx, sqlq, batchID, opts.EventType, opts.EventPayload)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) CreateBatchImageItem(ctx context.Context, params service.CreateBatchImageItemParams) (*service.BatchImageItem, error) {
|
||||
item, err := createBatchImageItemWithSQL(ctx, r.sql, params)
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) BulkCreateBatchImageItems(ctx context.Context, params []service.CreateBatchImageItemParams) error {
|
||||
if len(params) == 0 {
|
||||
return nil
|
||||
}
|
||||
if r.db == nil {
|
||||
for _, param := range params {
|
||||
if _, err := createBatchImageItemWithSQL(ctx, r.sql, param); err != nil {
|
||||
return translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
for _, param := range params {
|
||||
if _, err := createBatchImageItemWithSQL(ctx, tx, param); err != nil {
|
||||
return translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ReplaceBatchImageItemsForJob(ctx context.Context, batchID string, items []service.CreateBatchImageItemParams, counts service.BatchImageCounts) error {
|
||||
if r.db == nil {
|
||||
return r.replaceBatchImageItemsForJobWithSQL(ctx, r.sql, batchID, items, counts)
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
if err := r.replaceBatchImageItemsForJobWithSQL(ctx, tx, batchID, items, counts); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) replaceBatchImageItemsForJobWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID string, items []service.CreateBatchImageItemParams, counts service.BatchImageCounts) error {
|
||||
var id int64
|
||||
if err := sqlq.QueryRowContext(ctx, `SELECT id FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, batchID).Scan(&id); err != nil {
|
||||
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
|
||||
}
|
||||
if _, err := sqlq.ExecContext(ctx, `DELETE FROM batch_image_items WHERE job_id = $1`, batchID); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range items {
|
||||
item.JobID = batchID
|
||||
if _, err := createBatchImageItemWithSQL(ctx, sqlq, item); err != nil {
|
||||
return translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
|
||||
}
|
||||
}
|
||||
_, err := sqlq.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET success_count = $2,
|
||||
fail_count = $3,
|
||||
updated_at = $4
|
||||
WHERE batch_id = $1`, batchID, counts.SuccessCount, counts.FailCount, time.Now())
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ListBatchImageItems(ctx context.Context, batchID string, filter service.BatchImageItemFilter) ([]*service.BatchImageItem, error) {
|
||||
limit := filter.Limit
|
||||
if limit <= 0 || limit > 500 {
|
||||
limit = 100
|
||||
}
|
||||
if filter.Offset < 0 {
|
||||
filter.Offset = 0
|
||||
}
|
||||
|
||||
query := batchImageItemSelectSQL + " WHERE job_id = $1"
|
||||
args := []any{batchID}
|
||||
if filter.Status != "" {
|
||||
query += " AND status = $2"
|
||||
args = append(args, filter.Status)
|
||||
}
|
||||
query += " ORDER BY id ASC LIMIT $" + strconv.Itoa(len(args)+1) + " OFFSET $" + strconv.Itoa(len(args)+2)
|
||||
args = append(args, limit, filter.Offset)
|
||||
|
||||
rows, err := r.sql.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []*service.BatchImageItem
|
||||
for rows.Next() {
|
||||
item, err := scanBatchImageItem(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ListBatchImageItemsForOwner(ctx context.Context, userID, apiKeyID int64, batchID string, filter service.BatchImageItemFilter) ([]*service.BatchImageItem, error) {
|
||||
if _, err := r.GetBatchImageJobByBatchIDForOwner(ctx, userID, apiKeyID, batchID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.ListBatchImageItems(ctx, batchID, filter)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) GetBatchImageJobForDownload(ctx context.Context, userID, apiKeyID int64, batchID string) (*service.BatchImageJob, error) {
|
||||
return r.GetBatchImageJobByBatchIDForOwner(ctx, userID, apiKeyID, batchID)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) GetBatchImageItemForDownload(ctx context.Context, batchID, customID string) (*service.BatchImageItem, error) {
|
||||
item, err := scanBatchImageItem(r.sql.QueryRowContext(ctx, batchImageItemSelectSQL+`
|
||||
WHERE job_id = $1 AND custom_id = $2`, batchID, customID))
|
||||
if err != nil {
|
||||
return nil, translatePersistenceError(err, service.ErrBatchImageItemNotFound, nil)
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ListBatchImageItemsForDownload(ctx context.Context, batchID string, status string, limit int) ([]*service.BatchImageItem, error) {
|
||||
return r.ListBatchImageItems(ctx, batchID, service.BatchImageItemFilter{Status: status, Limit: limit})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ListBatchImageJobsDueForInputCleanup(ctx context.Context, cutoff time.Time, limit int) ([]*service.BatchImageJob, error) {
|
||||
if limit <= 0 || limit > 1000 {
|
||||
limit = 100
|
||||
}
|
||||
rows, err := r.sql.QueryContext(ctx, batchImageJobSelectSQL+`
|
||||
WHERE input_deleted_at IS NULL
|
||||
AND provider_input_ref IS NOT NULL
|
||||
AND status IN ('completed', 'failed', 'cancelled', 'output_deleted')
|
||||
AND COALESCE(finished_at, settled_at, updated_at, created_at) <= $1
|
||||
ORDER BY id ASC
|
||||
LIMIT $2`, cutoff, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanBatchImageJobs(rows)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) ListBatchImageJobsDueForOutputCleanup(ctx context.Context, now time.Time, limit int) ([]*service.BatchImageJob, error) {
|
||||
if limit <= 0 || limit > 1000 {
|
||||
limit = 100
|
||||
}
|
||||
rows, err := r.sql.QueryContext(ctx, batchImageJobSelectSQL+`
|
||||
WHERE output_deleted_at IS NULL
|
||||
AND provider_output_ref IS NOT NULL
|
||||
AND status = 'completed'
|
||||
AND output_expires_at IS NOT NULL
|
||||
AND output_expires_at <= $1
|
||||
ORDER BY output_expires_at ASC, id ASC
|
||||
LIMIT $2`, now, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanBatchImageJobs(rows)
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) MarkBatchImageInputDeleted(ctx context.Context, batchID string, deletedAt time.Time) error {
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET input_deleted_at = CASE WHEN input_deleted_at IS NULL THEN $2 ELSE input_deleted_at END,
|
||||
updated_at = $2,
|
||||
version = version + 1
|
||||
WHERE batch_id = $1`, batchID, deletedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
|
||||
return service.ErrBatchImageJobNotFound
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "input_cleanup_completed", map[string]any{
|
||||
"batch_id": batchID,
|
||||
"cleanup_target": "input",
|
||||
"deleted_at": deletedAt.UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) MarkBatchImageOutputDeleted(ctx context.Context, batchID string, deletedAt time.Time) error {
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET status = CASE WHEN status = 'completed' THEN 'output_deleted' ELSE status END,
|
||||
output_deleted_at = CASE WHEN output_deleted_at IS NULL THEN $2 ELSE output_deleted_at END,
|
||||
finished_at = CASE WHEN status = 'completed' AND finished_at IS NULL THEN $2 ELSE finished_at END,
|
||||
updated_at = $2,
|
||||
version = version + 1
|
||||
WHERE batch_id = $1`, batchID, deletedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
|
||||
return service.ErrBatchImageJobNotFound
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "output_cleanup_completed", map[string]any{
|
||||
"batch_id": batchID,
|
||||
"cleanup_target": "output",
|
||||
"deleted_at": deletedAt.UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) SetBatchImageOutputExpiresAt(ctx context.Context, batchID string, expiresAt time.Time) error {
|
||||
res, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET output_expires_at = CASE WHEN output_expires_at IS NULL THEN $2 ELSE output_expires_at END,
|
||||
updated_at = $3
|
||||
WHERE batch_id = $1`, batchID, expiresAt, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
|
||||
return service.ErrBatchImageJobNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) RecordBatchImageCleanupFailure(ctx context.Context, batchID, code, message string) error {
|
||||
_, err := r.sql.ExecContext(ctx, `
|
||||
UPDATE batch_image_jobs
|
||||
SET last_error_code = $2,
|
||||
last_error_message = $3,
|
||||
retry_count = retry_count + 1,
|
||||
updated_at = $4
|
||||
WHERE batch_id = $1`, batchID, code, message, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "output_cleanup_failed", map[string]any{"error_code": code})
|
||||
}
|
||||
|
||||
func (r *batchImageRepository) AppendBatchImageEvent(ctx context.Context, batchID, eventType string, payload any) error {
|
||||
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, eventType, payload)
|
||||
}
|
||||
|
||||
func createBatchImageJobWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.CreateBatchImageJobParams) (*service.BatchImageJob, error) {
|
||||
return scanBatchImageJob(sqlq.QueryRowContext(ctx, `
|
||||
INSERT INTO batch_image_jobs (
|
||||
batch_id, user_id, api_key_id, account_id, provider, model, status,
|
||||
provider_job_name, provider_input_ref, provider_output_ref, gcs_input_uri, gcs_output_uri,
|
||||
item_count, success_count, fail_count, cancelled_count,
|
||||
estimated_cost, hold_amount, actual_cost, currency, hold_id,
|
||||
idempotency_key, request_hash, manifest_hash, retry_count, output_expires_at
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7,
|
||||
$8, $9, $10, $11, $12,
|
||||
$13, $14, $15, $16,
|
||||
$17, $18, $19, $20, $21,
|
||||
$22, $23, $24, $25, $26
|
||||
)
|
||||
RETURNING `+batchImageJobColumns,
|
||||
params.BatchID, params.UserID, params.APIKeyID, params.AccountID, params.Provider, params.Model, params.Status,
|
||||
params.ProviderJobName, params.ProviderInputRef, params.ProviderOutputRef, params.GCSInputURI, params.GCSOutputURI,
|
||||
params.ItemCount, params.SuccessCount, params.FailCount, params.CancelledCount,
|
||||
params.EstimatedCost, params.HoldAmount, params.ActualCost, params.Currency, params.HoldID,
|
||||
params.IdempotencyKey, params.RequestHash, params.ManifestHash, params.RetryCount, params.OutputExpiresAt,
|
||||
))
|
||||
}
|
||||
|
||||
func createBatchImageItemWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.CreateBatchImageItemParams) (*service.BatchImageItem, error) {
|
||||
return scanBatchImageItem(sqlq.QueryRowContext(ctx, `
|
||||
INSERT INTO batch_image_items (
|
||||
job_id, custom_id, status, request_hash, prompt_preview, provider_source_object,
|
||||
source_line_number, source_byte_offset, source_byte_length,
|
||||
mime_type, file_extension, image_count,
|
||||
error_code, error_message, billed_amount, indexed_at
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6,
|
||||
$7, $8, $9,
|
||||
$10, $11, $12,
|
||||
$13, $14, $15, $16
|
||||
)
|
||||
RETURNING `+batchImageItemColumns,
|
||||
params.JobID, params.CustomID, params.Status, params.RequestHash, params.PromptPreview, params.ProviderSourceObject,
|
||||
params.SourceLineNumber, params.SourceByteOffset, params.SourceByteLength,
|
||||
params.MimeType, params.FileExtension, params.ImageCount,
|
||||
params.ErrorCode, params.ErrorMessage, params.BilledAmount, params.IndexedAt,
|
||||
))
|
||||
}
|
||||
|
||||
func appendBatchImageEventWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID, eventType string, payload any) error {
|
||||
var payloadArg any
|
||||
if payload != nil {
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
payloadArg = string(payloadBytes)
|
||||
}
|
||||
_, err := sqlq.ExecContext(ctx, `
|
||||
INSERT INTO batch_image_events (job_id, event_type, payload)
|
||||
VALUES ($1, $2, $3)`, batchID, eventType, payloadArg)
|
||||
return err
|
||||
}
|
||||
|
||||
type rowScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
const batchImageJobColumns = `
|
||||
id, batch_id, user_id, api_key_id, account_id, provider, model, status,
|
||||
provider_job_name, provider_input_ref, provider_output_ref, gcs_input_uri, gcs_output_uri,
|
||||
item_count, success_count, fail_count, cancelled_count,
|
||||
estimated_cost, hold_amount, actual_cost, currency, hold_id,
|
||||
idempotency_key, request_hash, manifest_hash,
|
||||
retry_count, version, output_expires_at, input_deleted_at, output_deleted_at,
|
||||
last_error_code, last_error_message,
|
||||
created_at, updated_at, submitted_at, started_at, finished_at, settled_at`
|
||||
|
||||
const batchImageJobSelectSQL = `SELECT ` + batchImageJobColumns + ` FROM batch_image_jobs`
|
||||
|
||||
func scanBatchImageJob(row rowScanner) (*service.BatchImageJob, error) {
|
||||
var job service.BatchImageJob
|
||||
var apiKeyID, accountID sql.NullInt64
|
||||
var providerJobName, providerInputRef, providerOutputRef, gcsInputURI, gcsOutputURI sql.NullString
|
||||
var holdAmount, actualCost sql.NullFloat64
|
||||
var holdID, idempotencyKey, requestHash, manifestHash sql.NullString
|
||||
var outputExpiresAt, inputDeletedAt, outputDeletedAt sql.NullTime
|
||||
var lastErrorCode, lastErrorMessage sql.NullString
|
||||
var submittedAt, startedAt, finishedAt, settledAt sql.NullTime
|
||||
|
||||
err := row.Scan(
|
||||
&job.ID, &job.BatchID, &job.UserID, &apiKeyID, &accountID, &job.Provider, &job.Model, &job.Status,
|
||||
&providerJobName, &providerInputRef, &providerOutputRef, &gcsInputURI, &gcsOutputURI,
|
||||
&job.ItemCount, &job.SuccessCount, &job.FailCount, &job.CancelledCount,
|
||||
&job.EstimatedCost, &holdAmount, &actualCost, &job.Currency, &holdID,
|
||||
&idempotencyKey, &requestHash, &manifestHash,
|
||||
&job.RetryCount, &job.Version, &outputExpiresAt, &inputDeletedAt, &outputDeletedAt,
|
||||
&lastErrorCode, &lastErrorMessage,
|
||||
&job.CreatedAt, &job.UpdatedAt, &submittedAt, &startedAt, &finishedAt, &settledAt,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
job.APIKeyID = batchImageNullInt64Ptr(apiKeyID)
|
||||
job.AccountID = batchImageNullInt64Ptr(accountID)
|
||||
job.ProviderJobName = batchImageNullStringPtr(providerJobName)
|
||||
job.ProviderInputRef = batchImageNullStringPtr(providerInputRef)
|
||||
job.ProviderOutputRef = batchImageNullStringPtr(providerOutputRef)
|
||||
job.GCSInputURI = batchImageNullStringPtr(gcsInputURI)
|
||||
job.GCSOutputURI = batchImageNullStringPtr(gcsOutputURI)
|
||||
job.HoldAmount = batchImageNullFloat64Ptr(holdAmount)
|
||||
job.ActualCost = batchImageNullFloat64Ptr(actualCost)
|
||||
job.HoldID = batchImageNullStringPtr(holdID)
|
||||
job.IdempotencyKey = batchImageNullStringPtr(idempotencyKey)
|
||||
job.RequestHash = batchImageNullStringPtr(requestHash)
|
||||
job.ManifestHash = batchImageNullStringPtr(manifestHash)
|
||||
job.OutputExpiresAt = batchImageNullTimePtr(outputExpiresAt)
|
||||
job.InputDeletedAt = batchImageNullTimePtr(inputDeletedAt)
|
||||
job.OutputDeletedAt = batchImageNullTimePtr(outputDeletedAt)
|
||||
job.LastErrorCode = batchImageNullStringPtr(lastErrorCode)
|
||||
job.LastErrorMessage = batchImageNullStringPtr(lastErrorMessage)
|
||||
job.SubmittedAt = batchImageNullTimePtr(submittedAt)
|
||||
job.StartedAt = batchImageNullTimePtr(startedAt)
|
||||
job.FinishedAt = batchImageNullTimePtr(finishedAt)
|
||||
job.SettledAt = batchImageNullTimePtr(settledAt)
|
||||
return &job, nil
|
||||
}
|
||||
|
||||
func scanBatchImageJobs(rows *sql.Rows) ([]*service.BatchImageJob, error) {
|
||||
var jobs []*service.BatchImageJob
|
||||
for rows.Next() {
|
||||
job, err := scanBatchImageJob(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
const batchImageItemColumns = `
|
||||
id, job_id, custom_id, status, request_hash, prompt_preview, provider_source_object,
|
||||
source_line_number, source_byte_offset, source_byte_length,
|
||||
mime_type, file_extension, image_count,
|
||||
error_code, error_message, billed_amount,
|
||||
created_at, indexed_at`
|
||||
|
||||
const batchImageItemSelectSQL = `SELECT ` + batchImageItemColumns + ` FROM batch_image_items`
|
||||
|
||||
func scanBatchImageItem(row rowScanner) (*service.BatchImageItem, error) {
|
||||
var item service.BatchImageItem
|
||||
var requestHash, promptPreview, providerSourceObject sql.NullString
|
||||
var sourceLineNumber sql.NullInt64
|
||||
var sourceByteOffset, sourceByteLength sql.NullInt64
|
||||
var mimeType, fileExtension, errorCode, errorMessage sql.NullString
|
||||
var billedAmount sql.NullFloat64
|
||||
var indexedAt sql.NullTime
|
||||
|
||||
err := row.Scan(
|
||||
&item.ID, &item.JobID, &item.CustomID, &item.Status, &requestHash, &promptPreview, &providerSourceObject,
|
||||
&sourceLineNumber, &sourceByteOffset, &sourceByteLength,
|
||||
&mimeType, &fileExtension, &item.ImageCount,
|
||||
&errorCode, &errorMessage, &billedAmount,
|
||||
&item.CreatedAt, &indexedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
item.RequestHash = batchImageNullStringPtr(requestHash)
|
||||
item.PromptPreview = batchImageNullStringPtr(promptPreview)
|
||||
item.ProviderSourceObject = batchImageNullStringPtr(providerSourceObject)
|
||||
item.SourceLineNumber = batchImageNullIntPtr(sourceLineNumber)
|
||||
item.SourceByteOffset = batchImageNullInt64Ptr(sourceByteOffset)
|
||||
item.SourceByteLength = batchImageNullInt64Ptr(sourceByteLength)
|
||||
item.MimeType = batchImageNullStringPtr(mimeType)
|
||||
item.FileExtension = batchImageNullStringPtr(fileExtension)
|
||||
item.ErrorCode = batchImageNullStringPtr(errorCode)
|
||||
item.ErrorMessage = batchImageNullStringPtr(errorMessage)
|
||||
item.BilledAmount = batchImageNullFloat64Ptr(billedAmount)
|
||||
item.IndexedAt = batchImageNullTimePtr(indexedAt)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
func batchImageNullStringPtr(v sql.NullString) *string {
|
||||
if !v.Valid {
|
||||
return nil
|
||||
}
|
||||
return &v.String
|
||||
}
|
||||
|
||||
func batchImageNullInt64Ptr(v sql.NullInt64) *int64 {
|
||||
if !v.Valid {
|
||||
return nil
|
||||
}
|
||||
return &v.Int64
|
||||
}
|
||||
|
||||
func batchImageNullIntPtr(v sql.NullInt64) *int {
|
||||
if !v.Valid {
|
||||
return nil
|
||||
}
|
||||
i := int(v.Int64)
|
||||
return &i
|
||||
}
|
||||
|
||||
func batchImageNullFloat64Ptr(v sql.NullFloat64) *float64 {
|
||||
if !v.Valid {
|
||||
return nil
|
||||
}
|
||||
return &v.Float64
|
||||
}
|
||||
|
||||
func batchImageNullTimePtr(v sql.NullTime) *time.Time {
|
||||
if !v.Valid {
|
||||
return nil
|
||||
}
|
||||
return &v.Time
|
||||
}
|
||||
|
||||
var _ service.BatchImageRepository = (*batchImageRepository)(nil)
|
||||
@@ -0,0 +1,339 @@
|
||||
//go:build integration
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBatchImageRepository_CreateJobAndDuplicates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "create")
|
||||
|
||||
job, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 2,
|
||||
EstimatedCost: 0.02,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, batchID, job.BatchID)
|
||||
require.Equal(t, service.BatchImageJobStatusCreated, job.Status)
|
||||
require.Equal(t, "USD", job.Currency)
|
||||
|
||||
_, err = repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, service.ErrBatchImageJobExists))
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_InvalidProvider(t *testing.T) {
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
|
||||
_, err := repo.CreateBatchImageJob(context.Background(), service.CreateBatchImageJobParams{
|
||||
BatchID: batchImageTestID(t, "provider"),
|
||||
UserID: 1001,
|
||||
Provider: "unknown",
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, service.ErrBatchImageInvalidProvider))
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_TransitionIncrementsVersionAndEvents(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "transition")
|
||||
now := time.Date(2026, 7, 3, 8, 0, 0, 0, time.UTC)
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderVertex,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusUploading, service.BatchImageTransitionOptions{
|
||||
EventType: "status_changed",
|
||||
EventPayload: map[string]any{"to": service.BatchImageJobStatusUploading},
|
||||
Now: &now,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, service.BatchImageJobStatusUploading, job.Status)
|
||||
require.Equal(t, 1, job.Version)
|
||||
|
||||
var eventCount int
|
||||
err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM batch_image_events WHERE job_id = $1 AND event_type = 'status_changed'`, batchID).Scan(&eventCount)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, eventCount)
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_InvalidTransition(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "invalid-transition")
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusRunning, service.BatchImageTransitionOptions{})
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, service.ErrBatchImageInvalidTransition))
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_TerminalStatusCannotMoveBack(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "terminal")
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
Status: service.BatchImageJobStatusCompleted,
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusRunning, service.BatchImageTransitionOptions{})
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, service.ErrBatchImageInvalidTransition))
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_ItemCustomIDUniqueness(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
firstBatchID := batchImageTestID(t, "items-a")
|
||||
secondBatchID := batchImageTestID(t, "items-b")
|
||||
|
||||
for _, batchID := range []string{firstBatchID, secondBatchID} {
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
_, err := repo.CreateBatchImageItem(ctx, service.CreateBatchImageItemParams{
|
||||
JobID: firstBatchID,
|
||||
CustomID: "line-1",
|
||||
Status: service.BatchImageItemStatusSuccess,
|
||||
ImageCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = repo.CreateBatchImageItem(ctx, service.CreateBatchImageItemParams{
|
||||
JobID: firstBatchID,
|
||||
CustomID: "line-1",
|
||||
Status: service.BatchImageItemStatusFailed,
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.True(t, errors.Is(err, service.ErrBatchImageItemExists))
|
||||
|
||||
_, err = repo.CreateBatchImageItem(ctx, service.CreateBatchImageItemParams{
|
||||
JobID: secondBatchID,
|
||||
CustomID: "line-1",
|
||||
Status: service.BatchImageItemStatusSuccess,
|
||||
ImageCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
items, err := repo.ListBatchImageItems(ctx, firstBatchID, service.BatchImageItemFilter{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, items, 1)
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_ReplaceBatchImageItemsForJob(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "replace-items")
|
||||
lineOne := 1
|
||||
lineTwo := 2
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 2,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.ReplaceBatchImageItemsForJob(ctx, batchID, []service.CreateBatchImageItemParams{
|
||||
{CustomID: "old", Status: service.BatchImageItemStatusSuccess, SourceLineNumber: &lineOne, ImageCount: 1},
|
||||
}, service.BatchImageCounts{SuccessCount: 1})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.ReplaceBatchImageItemsForJob(ctx, batchID, []service.CreateBatchImageItemParams{
|
||||
{CustomID: "new-ok", Status: service.BatchImageItemStatusSuccess, SourceLineNumber: &lineOne, ImageCount: 1},
|
||||
{CustomID: "new-fail", Status: service.BatchImageItemStatusFailed, SourceLineNumber: &lineTwo, ErrorCode: batchImageTestStringPtr("SAFETY_BLOCKED")},
|
||||
}, service.BatchImageCounts{SuccessCount: 1, FailCount: 1})
|
||||
require.NoError(t, err)
|
||||
|
||||
items, err := repo.ListBatchImageItems(ctx, batchID, service.BatchImageItemFilter{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, items, 2)
|
||||
require.Equal(t, "new-ok", items[0].CustomID)
|
||||
require.Equal(t, "new-fail", items[1].CustomID)
|
||||
|
||||
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, job.SuccessCount)
|
||||
require.Equal(t, 1, job.FailCount)
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_MarkBatchImageJobSettled(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "settled")
|
||||
apiKeyID := int64(2001)
|
||||
accountID := int64(3001)
|
||||
providerJob := "providers/job"
|
||||
outputRef := "files/output"
|
||||
now := time.Date(2026, 7, 4, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
APIKeyID: &apiKeyID,
|
||||
AccountID: &accountID,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-image",
|
||||
Status: service.BatchImageJobStatusSettling,
|
||||
ProviderJobName: &providerJob,
|
||||
ProviderOutputRef: &outputRef,
|
||||
ItemCount: 3,
|
||||
SuccessCount: 2,
|
||||
FailCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.MarkBatchImageJobSettled(ctx, service.MarkBatchImageJobSettledParams{
|
||||
BatchID: batchID,
|
||||
ActualCost: 0.5,
|
||||
ManifestHash: "manifest-hash",
|
||||
EventPayload: map[string]any{"request_id": "batch_image_settlement:" + batchID},
|
||||
Now: &now,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, service.BatchImageJobStatusCompleted, job.Status)
|
||||
require.NotNil(t, job.ActualCost)
|
||||
require.Equal(t, 0.5, *job.ActualCost)
|
||||
require.Equal(t, "manifest-hash", batchImageDerefTest(job.ManifestHash))
|
||||
require.NotNil(t, job.SettledAt)
|
||||
require.Equal(t, now, *job.SettledAt)
|
||||
|
||||
var eventCount int
|
||||
err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM batch_image_events WHERE job_id = $1 AND event_type = 'settlement_completed'`, batchID).Scan(&eventCount)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, eventCount)
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_SetBatchImageJobSettlementFailed(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "settlement-failed")
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderGeminiAPI,
|
||||
Model: "gemini-image",
|
||||
Status: service.BatchImageJobStatusSettling,
|
||||
ItemCount: 1,
|
||||
SuccessCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.SetBatchImageJobSettlementFailed(ctx, batchID, "SETTLEMENT_BILLING_FAILED", "temporary")
|
||||
require.NoError(t, err)
|
||||
|
||||
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, service.BatchImageJobStatusSettling, job.Status)
|
||||
require.Equal(t, "SETTLEMENT_BILLING_FAILED", batchImageDerefTest(job.LastErrorCode))
|
||||
require.Equal(t, "temporary", batchImageDerefTest(job.LastErrorMessage))
|
||||
require.Equal(t, 1, job.RetryCount)
|
||||
}
|
||||
|
||||
func TestBatchImageRepository_AppendEvent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tx := testTx(t)
|
||||
repo := newBatchImageRepositoryWithSQL(tx)
|
||||
batchID := batchImageTestID(t, "event")
|
||||
|
||||
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
|
||||
BatchID: batchID,
|
||||
UserID: 1001,
|
||||
Provider: service.BatchImageProviderVertex,
|
||||
Model: "gemini-2.5-flash-image",
|
||||
ItemCount: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = repo.AppendBatchImageEvent(ctx, batchID, "job_created", map[string]any{"batch_id": batchID})
|
||||
require.NoError(t, err)
|
||||
|
||||
var payload string
|
||||
err = tx.QueryRowContext(ctx, `SELECT payload::text FROM batch_image_events WHERE job_id = $1 AND event_type = 'job_created'`, batchID).Scan(&payload)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, payload, batchID)
|
||||
}
|
||||
|
||||
func batchImageTestID(t *testing.T, prefix string) string {
|
||||
t.Helper()
|
||||
return "imgbatch_" + uniqueTestValue(t, prefix)
|
||||
}
|
||||
|
||||
func batchImageTestStringPtr(v string) *string {
|
||||
return &v
|
||||
}
|
||||
|
||||
func batchImageDerefTest(v *string) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return *v
|
||||
}
|
||||
@@ -77,6 +77,7 @@ var ProviderSet = wire.NewSet(
|
||||
NewAnnouncementReadRepository,
|
||||
NewUsageLogRepository,
|
||||
NewUsageBillingRepository,
|
||||
NewBatchImageRepository,
|
||||
NewIdempotencyRepository,
|
||||
NewUsageCleanupRepository,
|
||||
NewDashboardAggregationRepository,
|
||||
@@ -115,6 +116,8 @@ var ProviderSet = wire.NewSet(
|
||||
NewRedeemCache,
|
||||
NewUpdateCache,
|
||||
NewGeminiTokenCache,
|
||||
NewBatchImageQueue,
|
||||
NewBatchImageDownloadLimiter,
|
||||
NewLeaderLockCache,
|
||||
ProvideSchedulerCache,
|
||||
NewSchedulerOutboxRepository,
|
||||
|
||||
Reference in new issue
Block a user