feat: add batch image MVP

This commit is contained in:
Turtle_Li committed 2026-07-04 05:30:50 +08:00
1 parent 28c2d20d9f
commit a994fbd77a
79 files changed
+34113 -36

No files matched your search

@@ -0,0 +1,112 @@
package repository
import (
"context"
"sync"
"time"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/redis/go-redis/v9"
)
const (
defaultBatchImageDownloadActivePrefix = "batch_image:download:active:"
defaultBatchImageDownloadActiveTTL = 10 * time.Minute
defaultBatchImageDownloadConcurrency = 2
)
var batchImageDownloadAcquireScript = redis.NewScript(`
local current = tonumber(redis.call("GET", KEYS[1]) or "0")
local max = tonumber(ARGV[1])
if current >= max then
return 0
end
redis.call("INCR", KEYS[1])
redis.call("EXPIRE", KEYS[1], ARGV[2])
return 1
`)
var batchImageDownloadReleaseScript = redis.NewScript(`
local current = tonumber(redis.call("GET", KEYS[1]) or "0")
if current <= 1 then
redis.call("DEL", KEYS[1])
return 0
end
return redis.call("DECR", KEYS[1])
`)
type batchImageDownloadLimiter struct {
rdb *redis.Client
activePrefix string
maxActive int
ttl time.Duration
}
func NewBatchImageDownloadLimiter(rdb *redis.Client, cfg *config.Config) service.BatchImageDownloadLimiter {
maxActive := defaultBatchImageDownloadConcurrency
ttl := defaultBatchImageDownloadActiveTTL
if cfg != nil {
if cfg.BatchImage.MaxDownloadConcurrencyPerUser > 0 {
maxActive = cfg.BatchImage.MaxDownloadConcurrencyPerUser
}
if cfg.BatchImage.MaxDownloadDurationSeconds > 0 {
ttl = time.Duration(cfg.BatchImage.MaxDownloadDurationSeconds) * time.Second
}
}
return &batchImageDownloadLimiter{
rdb: rdb,
activePrefix: defaultBatchImageDownloadActivePrefix,
maxActive: maxActive,
ttl: ttl,
}
}
func newBatchImageDownloadLimiterForTest(rdb *redis.Client, maxActive int, ttl time.Duration) *batchImageDownloadLimiter {
if maxActive <= 0 {
maxActive = defaultBatchImageDownloadConcurrency
}
if ttl <= 0 {
ttl = defaultBatchImageDownloadActiveTTL
}
return &batchImageDownloadLimiter{rdb: rdb, activePrefix: defaultBatchImageDownloadActivePrefix, maxActive: maxActive, ttl: ttl}
}
func (l *batchImageDownloadLimiter) Acquire(ctx context.Context, userID string, kind string) (service.BatchImageDownloadPermit, error) {
if l == nil || l.rdb == nil {
return nil, service.ErrBatchImageDownloadLimited
}
key := l.activeKey(userID)
ok, err := batchImageDownloadAcquireScript.Run(ctx, l.rdb, []string{key}, l.maxActive, int(l.ttl.Seconds())).Int()
if err != nil {
return nil, err
}
if ok != 1 {
return nil, service.ErrBatchImageDownloadLimited
}
return &batchImageDownloadPermit{rdb: l.rdb, key: key}, nil
}
func (l *batchImageDownloadLimiter) activeKey(userID string) string {
return l.activePrefix + userID
}
type batchImageDownloadPermit struct {
rdb *redis.Client
key string
once sync.Once
err error
}
func (p *batchImageDownloadPermit) Release(ctx context.Context) error {
if p == nil || p.rdb == nil || p.key == "" {
return nil
}
p.once.Do(func() {
_, p.err = batchImageDownloadReleaseScript.Run(ctx, p.rdb, []string{p.key}).Result()
})
return p.err
}
var _ service.BatchImageDownloadLimiter = (*batchImageDownloadLimiter)(nil)
var _ service.BatchImageDownloadPermit = (*batchImageDownloadPermit)(nil)
@@ -0,0 +1,38 @@
//go:build unit
package repository
import (
"context"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/require"
)
func TestBatchImageDownloadLimiter_AcquireDenyReleaseAndTTL(t *testing.T) {
ctx := context.Background()
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
t.Cleanup(func() { _ = rdb.Close() })
limiter := newBatchImageDownloadLimiterForTest(rdb, 1, time.Minute)
permit, err := limiter.Acquire(ctx, "11", "zip")
require.NoError(t, err)
require.NotNil(t, permit)
require.True(t, mr.TTL(limiter.activeKey("11")) > 0)
_, err = limiter.Acquire(ctx, "11", "zip")
require.ErrorIs(t, err, service.ErrBatchImageDownloadLimited)
require.NoError(t, permit.Release(ctx))
require.NoError(t, permit.Release(ctx))
require.False(t, mr.Exists(limiter.activeKey("11")))
permit, err = limiter.Acquire(ctx, "11", "zip")
require.NoError(t, err)
require.NotNil(t, permit)
}
@@ -0,0 +1,280 @@
package repository
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"time"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/redis/go-redis/v9"
)
const (
defaultBatchImageReadyKey = "batch_image:queue:ready"
defaultBatchImageDelayedKey = "batch_image:queue:delayed"
defaultBatchImageActiveKey = "batch_image:queue:active"
defaultBatchImageInflightPrefix = "batch_image:queue:inflight:"
defaultBatchImageLockPrefix = "batch_image:queue:lock:"
defaultBatchImageInflightTTL = 7 * 24 * time.Hour
defaultBatchImageJobLockTTL = 5 * time.Minute
)
var batchImageMoveDueDelayedScript = redis.NewScript(`
local jobs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, ARGV[2])
for _, job in ipairs(jobs) do
redis.call("ZREM", KEYS[1], job)
redis.call("LPUSH", KEYS[2], job)
end
return #jobs
`)
var batchImageRecoverStaleActiveScript = redis.NewScript(`
local jobs = redis.call("ZRANGEBYSCORE", KEYS[1], "-inf", ARGV[1], "LIMIT", 0, ARGV[2])
for _, job in ipairs(jobs) do
redis.call("ZREM", KEYS[1], job)
redis.call("LPUSH", KEYS[2], job)
end
return #jobs
`)
var batchImageReleaseLockScript = redis.NewScript(`
if redis.call("GET", KEYS[1]) == ARGV[1] then
return redis.call("DEL", KEYS[1])
end
return 0
`)
type batchImageQueue struct {
rdb *redis.Client
readyKey string
delayedKey string
activeKey string
inflightPrefix string
lockPrefix string
inflightTTL time.Duration
lockTTL time.Duration
}
func NewBatchImageQueue(rdb *redis.Client, cfg *config.Config) service.BatchImageQueue {
return newBatchImageQueueWithOptions(rdb, batchImageQueueOptionsFromConfig(cfg))
}
type batchImageQueueOptions struct {
ReadyKey string
DelayedKey string
ActiveKey string
InflightPrefix string
LockPrefix string
InflightTTL time.Duration
LockTTL time.Duration
}
func newBatchImageQueueWithOptions(rdb *redis.Client, opts batchImageQueueOptions) *batchImageQueue {
opts = normalizeBatchImageQueueOptions(opts)
return &batchImageQueue{
rdb: rdb,
readyKey: opts.ReadyKey,
delayedKey: opts.DelayedKey,
activeKey: opts.ActiveKey,
inflightPrefix: opts.InflightPrefix,
lockPrefix: opts.LockPrefix,
inflightTTL: opts.InflightTTL,
lockTTL: opts.LockTTL,
}
}
func batchImageQueueOptionsFromConfig(cfg *config.Config) batchImageQueueOptions {
if cfg == nil {
return batchImageQueueOptions{}
}
return batchImageQueueOptions{
ReadyKey: cfg.BatchImage.QueueReadyKey,
DelayedKey: cfg.BatchImage.QueueDelayedKey,
ActiveKey: cfg.BatchImage.QueueActiveKey,
InflightPrefix: cfg.BatchImage.InflightKeyPrefix,
LockPrefix: cfg.BatchImage.LockKeyPrefix,
InflightTTL: time.Duration(cfg.BatchImage.InflightTTLSeconds) * time.Second,
LockTTL: time.Duration(cfg.BatchImage.JobLockTTLSeconds) * time.Second,
}
}
func normalizeBatchImageQueueOptions(opts batchImageQueueOptions) batchImageQueueOptions {
if opts.ReadyKey == "" {
opts.ReadyKey = defaultBatchImageReadyKey
}
if opts.DelayedKey == "" {
opts.DelayedKey = defaultBatchImageDelayedKey
}
if opts.ActiveKey == "" {
opts.ActiveKey = defaultBatchImageActiveKey
}
if opts.InflightPrefix == "" {
opts.InflightPrefix = defaultBatchImageInflightPrefix
}
if opts.LockPrefix == "" {
opts.LockPrefix = defaultBatchImageLockPrefix
}
if opts.InflightTTL <= 0 {
opts.InflightTTL = defaultBatchImageInflightTTL
}
if opts.LockTTL <= 0 {
opts.LockTTL = defaultBatchImageJobLockTTL
}
return opts
}
func (q *batchImageQueue) Enqueue(ctx context.Context, batchID string) error {
if !service.IsValidBatchImageID(batchID) {
return service.ErrInvalidBatchImageQueuePayload
}
ok, err := q.rdb.SetNX(ctx, q.inflightKey(batchID), batchID, q.inflightTTL).Result()
if err != nil {
return err
}
if !ok {
return service.ErrBatchImageAlreadyQueued
}
if err := q.rdb.LPush(ctx, q.readyKey, batchID).Err(); err != nil {
_ = q.rdb.Del(ctx, q.inflightKey(batchID)).Err()
return err
}
return nil
}
func (q *batchImageQueue) Reserve(ctx context.Context, blockTimeout time.Duration) (service.ReservedBatchImageJob, error) {
result, err := q.rdb.BRPop(ctx, blockTimeout, q.readyKey).Result()
if errors.Is(err, redis.Nil) {
return service.ReservedBatchImageJob{}, service.ErrBatchImageQueueEmpty
}
if err != nil {
return service.ReservedBatchImageJob{}, err
}
if len(result) != 2 || !service.IsValidBatchImageID(result[1]) {
return service.ReservedBatchImageJob{}, service.ErrInvalidBatchImageQueuePayload
}
batchID := result[1]
if err := q.rdb.ZAdd(ctx, q.activeKey, redis.Z{
Score: float64(time.Now().UnixMilli()),
Member: batchID,
}).Err(); err != nil {
return service.ReservedBatchImageJob{}, err
}
return service.ReservedBatchImageJob{BatchID: batchID}, nil
}
func (q *batchImageQueue) RequeueAfter(ctx context.Context, batchID string, delay time.Duration) error {
if !service.IsValidBatchImageID(batchID) {
return service.ErrInvalidBatchImageQueuePayload
}
pipe := q.rdb.TxPipeline()
pipe.ZRem(ctx, q.activeKey, batchID)
pipe.ZRem(ctx, q.delayedKey, batchID)
if delay <= 0 {
pipe.LPush(ctx, q.readyKey, batchID)
} else {
pipe.ZAdd(ctx, q.delayedKey, redis.Z{
Score: float64(time.Now().Add(delay).UnixMilli()),
Member: batchID,
})
}
_, err := pipe.Exec(ctx)
return err
}
func (q *batchImageQueue) Ack(ctx context.Context, batchID string) error {
if !service.IsValidBatchImageID(batchID) {
return service.ErrInvalidBatchImageQueuePayload
}
pipe := q.rdb.TxPipeline()
pipe.ZRem(ctx, q.activeKey, batchID)
pipe.ZRem(ctx, q.delayedKey, batchID)
pipe.Del(ctx, q.inflightKey(batchID))
_, err := pipe.Exec(ctx)
return err
}
func (q *batchImageQueue) Heartbeat(ctx context.Context, batchID string) error {
if !service.IsValidBatchImageID(batchID) {
return service.ErrInvalidBatchImageQueuePayload
}
return q.rdb.ZAdd(ctx, q.activeKey, redis.Z{
Score: float64(time.Now().UnixMilli()),
Member: batchID,
}).Err()
}
func (q *batchImageQueue) MoveDueDelayedToReady(ctx context.Context, limit int) (int, error) {
if limit <= 0 {
limit = 100
}
return batchImageMoveDueDelayedScript.Run(ctx, q.rdb, []string{q.delayedKey, q.readyKey}, time.Now().UnixMilli(), limit).Int()
}
func (q *batchImageQueue) RecoverStaleActive(ctx context.Context, staleAfter time.Duration, limit int) (int, error) {
if staleAfter <= 0 {
return 0, service.ErrInvalidBatchImageQueuePayload
}
if limit <= 0 {
limit = 100
}
cutoff := time.Now().Add(-staleAfter).UnixMilli()
return batchImageRecoverStaleActiveScript.Run(ctx, q.rdb, []string{q.activeKey, q.readyKey}, cutoff, limit).Int()
}
func (q *batchImageQueue) TryAcquireJobLock(ctx context.Context, batchID string, ttl time.Duration) (service.BatchImageJobLock, bool, error) {
if !service.IsValidBatchImageID(batchID) {
return nil, false, service.ErrInvalidBatchImageQueuePayload
}
if ttl <= 0 {
ttl = q.lockTTL
}
token, err := newBatchImageLockToken()
if err != nil {
return nil, false, err
}
key := q.lockKey(batchID)
ok, err := q.rdb.SetNX(ctx, key, token, ttl).Result()
if err != nil {
return nil, false, err
}
if !ok {
return nil, false, nil
}
return &batchImageRedisJobLock{rdb: q.rdb, key: key, token: token}, true, nil
}
func (q *batchImageQueue) inflightKey(batchID string) string {
return q.inflightPrefix + batchID
}
func (q *batchImageQueue) lockKey(batchID string) string {
return q.lockPrefix + batchID
}
type batchImageRedisJobLock struct {
rdb *redis.Client
key string
token string
}
func (l *batchImageRedisJobLock) Release(ctx context.Context) error {
if l == nil || l.rdb == nil || l.key == "" || l.token == "" {
return nil
}
return batchImageReleaseLockScript.Run(ctx, l.rdb, []string{l.key}, l.token).Err()
}
func newBatchImageLockToken() (string, error) {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
return "", err
}
return hex.EncodeToString(b[:]), nil
}
var _ service.BatchImageQueue = (*batchImageQueue)(nil)
@@ -0,0 +1,123 @@
//go:build unit
package repository
import (
"context"
"errors"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/require"
)
func TestBatchImageQueue_DuplicateEnqueueReturnsAlreadyQueued(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
batchID := "imgbatch_duplicate"
require.NoError(t, queue.Enqueue(ctx, batchID))
err := queue.Enqueue(ctx, batchID)
require.Error(t, err)
require.True(t, errors.Is(err, service.ErrBatchImageAlreadyQueued))
}
func TestBatchImageQueue_RequeueAfterMovesJobFromActiveToDelayed(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
batchID := "imgbatch_requeue_after"
require.NoError(t, queue.rdb.ZAdd(ctx, queue.activeKey, redis.Z{
Score: float64(time.Now().UnixMilli()),
Member: batchID,
}).Err())
require.NoError(t, queue.RequeueAfter(ctx, batchID, time.Minute))
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, batchID).Err(), redis.Nil)
score, err := queue.rdb.ZScore(ctx, queue.delayedKey, batchID).Result()
require.NoError(t, err)
require.Greater(t, score, float64(time.Now().UnixMilli()))
}
func TestBatchImageQueue_MoveDueDelayedToReadyMovesDueJobs(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
dueBatchID := "imgbatch_due"
futureBatchID := "imgbatch_future"
now := time.Now()
require.NoError(t, queue.rdb.ZAdd(ctx, queue.delayedKey,
redis.Z{Score: float64(now.Add(-time.Second).UnixMilli()), Member: dueBatchID},
redis.Z{Score: float64(now.Add(time.Hour).UnixMilli()), Member: futureBatchID},
).Err())
moved, err := queue.MoveDueDelayedToReady(ctx, 10)
require.NoError(t, err)
require.Equal(t, 1, moved)
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.delayedKey, dueBatchID).Err(), redis.Nil)
require.NoError(t, queue.rdb.ZScore(ctx, queue.delayedKey, futureBatchID).Err())
reserved, err := queue.Reserve(ctx, time.Millisecond)
require.NoError(t, err)
require.Equal(t, dueBatchID, reserved.BatchID)
}
func TestBatchImageQueue_RecoverStaleActiveMovesStaleJobsToReady(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
staleBatchID := "imgbatch_stale"
recentBatchID := "imgbatch_recent"
now := time.Now()
require.NoError(t, queue.rdb.ZAdd(ctx, queue.activeKey,
redis.Z{Score: float64(now.Add(-time.Hour).UnixMilli()), Member: staleBatchID},
redis.Z{Score: float64(now.UnixMilli()), Member: recentBatchID},
).Err())
moved, err := queue.RecoverStaleActive(ctx, 10*time.Minute, 10)
require.NoError(t, err)
require.Equal(t, 1, moved)
require.ErrorIs(t, queue.rdb.ZScore(ctx, queue.activeKey, staleBatchID).Err(), redis.Nil)
require.NoError(t, queue.rdb.ZScore(ctx, queue.activeKey, recentBatchID).Err())
reserved, err := queue.Reserve(ctx, time.Millisecond)
require.NoError(t, err)
require.Equal(t, staleBatchID, reserved.BatchID)
}
func TestBatchImageQueue_JobLockReleaseOnlyDeletesMatchingToken(t *testing.T) {
ctx := context.Background()
queue, _ := newBatchImageQueueTest(t)
batchID := "imgbatch_lock"
lock, ok, err := queue.TryAcquireJobLock(ctx, batchID, time.Minute)
require.NoError(t, err)
require.True(t, ok)
require.NoError(t, queue.rdb.Set(ctx, queue.lockKey(batchID), "other-token", time.Minute).Err())
require.NoError(t, lock.Release(ctx))
got, err := queue.rdb.Get(ctx, queue.lockKey(batchID)).Result()
require.NoError(t, err)
require.Equal(t, "other-token", got)
require.NoError(t, queue.rdb.Del(ctx, queue.lockKey(batchID)).Err())
lock, ok, err = queue.TryAcquireJobLock(ctx, batchID, time.Minute)
require.NoError(t, err)
require.True(t, ok)
require.NoError(t, lock.Release(ctx))
require.ErrorIs(t, queue.rdb.Get(ctx, queue.lockKey(batchID)).Err(), redis.Nil)
}
func newBatchImageQueueTest(t *testing.T) (*batchImageQueue, *miniredis.Miniredis) {
t.Helper()
mr := miniredis.RunT(t)
rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
t.Cleanup(func() {
_ = rdb.Close()
})
queue := newBatchImageQueueWithOptions(rdb, batchImageQueueOptions{
InflightTTL: time.Hour,
LockTTL: time.Minute,
})
return queue, mr
}
@@ -0,0 +1,782 @@
package repository
import (
"context"
"database/sql"
"encoding/json"
"strconv"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
)
type batchImageSQLExecutor interface {
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
}
type batchImageRepository struct {
db *sql.DB
sql batchImageSQLExecutor
}
func NewBatchImageRepository(db *sql.DB) service.BatchImageRepository {
return &batchImageRepository{db: db, sql: db}
}
func newBatchImageRepositoryWithSQL(sqlq batchImageSQLExecutor) *batchImageRepository {
return &batchImageRepository{sql: sqlq}
}
func (r *batchImageRepository) CreateBatchImageJob(ctx context.Context, params service.CreateBatchImageJobParams) (*service.BatchImageJob, error) {
if !service.IsSupportedBatchImageProvider(params.Provider) {
return nil, service.ErrBatchImageInvalidProvider
}
if params.BatchID == "" {
batchID, err := service.NewBatchImageID()
if err != nil {
return nil, err
}
params.BatchID = batchID
}
if params.Status == "" {
params.Status = service.BatchImageJobStatusCreated
}
if params.Currency == "" {
params.Currency = "USD"
}
job, err := createBatchImageJobWithSQL(ctx, r.sql, params)
if err != nil {
return nil, translatePersistenceError(err, nil, service.ErrBatchImageJobExists)
}
return job, nil
}
func (r *batchImageRepository) GetBatchImageJobByBatchID(ctx context.Context, batchID string) (*service.BatchImageJob, error) {
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+" WHERE batch_id = $1", batchID))
if err != nil {
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
return job, nil
}
func (r *batchImageRepository) GetBatchImageJobByIdempotencyKey(ctx context.Context, userID, apiKeyID int64, key string) (*service.BatchImageJob, error) {
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+`
WHERE user_id = $1 AND api_key_id = $2 AND idempotency_key = $3
ORDER BY id DESC LIMIT 1`, userID, apiKeyID, key))
if err != nil {
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
return job, nil
}
func (r *batchImageRepository) GetBatchImageJobByBatchIDForOwner(ctx context.Context, userID, apiKeyID int64, batchID string) (*service.BatchImageJob, error) {
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+`
WHERE batch_id = $1 AND user_id = $2 AND api_key_id = $3`, batchID, userID, apiKeyID))
if err != nil {
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
return job, nil
}
func (r *batchImageRepository) GetBatchImageJobByID(ctx context.Context, id int64) (*service.BatchImageJob, error) {
job, err := scanBatchImageJob(r.sql.QueryRowContext(ctx, batchImageJobSelectSQL+" WHERE id = $1", id))
if err != nil {
return nil, translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
return job, nil
}
func (r *batchImageRepository) TransitionBatchImageJobStatus(ctx context.Context, batchID, toStatus string, opts service.BatchImageTransitionOptions) error {
if r.db == nil {
return r.transitionBatchImageJobStatusWithSQL(ctx, r.sql, batchID, toStatus, opts)
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
if err := r.transitionBatchImageJobStatusWithSQL(ctx, tx, batchID, toStatus, opts); err != nil {
return err
}
return tx.Commit()
}
func (r *batchImageRepository) UpdateBatchImageJobProviderOutputRef(ctx context.Context, batchID, providerOutputRef string) error {
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET provider_output_ref = $2, updated_at = $3
WHERE batch_id = $1`, batchID, providerOutputRef, time.Now())
if err != nil {
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
return service.ErrBatchImageJobNotFound
}
return nil
}
func (r *batchImageRepository) UpdateBatchImageJobProviderSubmit(ctx context.Context, params service.UpdateBatchImageJobProviderSubmitParams) error {
if r.db == nil {
return r.updateBatchImageJobProviderSubmitWithSQL(ctx, r.sql, params)
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
if err := r.updateBatchImageJobProviderSubmitWithSQL(ctx, tx, params); err != nil {
return err
}
return tx.Commit()
}
func (r *batchImageRepository) updateBatchImageJobProviderSubmitWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.UpdateBatchImageJobProviderSubmitParams) error {
var current string
if err := sqlq.QueryRowContext(ctx, `SELECT status FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, params.BatchID).Scan(&current); err != nil {
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
if !service.CanTransitionBatchImageJob(current, service.BatchImageJobStatusSubmitted) {
return service.ErrBatchImageInvalidTransition
}
now := time.Now()
if _, err := sqlq.ExecContext(ctx, `
UPDATE batch_image_jobs
SET status = 'submitted',
provider_job_name = $2,
provider_input_ref = NULLIF($3, ''),
provider_output_ref = NULLIF($4, ''),
gcs_input_uri = NULLIF($5, ''),
gcs_output_uri = NULLIF($6, ''),
submitted_at = CASE WHEN submitted_at IS NULL THEN $7 ELSE submitted_at END,
updated_at = $7,
version = version + 1
WHERE batch_id = $1`, params.BatchID, params.ProviderJobName, params.ProviderInputRef, params.ProviderOutputRef, params.GCSInputURI, params.GCSOutputURI, now); err != nil {
return err
}
return appendBatchImageEventWithSQL(ctx, sqlq, params.BatchID, "provider_submitted", params.EventPayload)
}
func (r *batchImageRepository) RecordBatchImageJobSubmitFailure(ctx context.Context, batchID, code, message string, markFailed bool) error {
now := time.Now()
statusSQL := "status"
if markFailed {
statusSQL = "'failed'"
}
_, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET status = `+statusSQL+`,
last_error_code = $2,
last_error_message = $3,
finished_at = CASE WHEN `+statusSQL+` = 'failed' AND finished_at IS NULL THEN $4 ELSE finished_at END,
updated_at = $4,
version = version + 1
WHERE batch_id = $1`, batchID, code, message, now)
if err != nil {
return err
}
eventType := "submit_failed"
if !markFailed {
eventType = "queue_failed"
}
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, eventType, map[string]any{"error_code": code})
}
func (r *batchImageRepository) MarkBatchImageJobSettled(ctx context.Context, params service.MarkBatchImageJobSettledParams) error {
if r.db == nil {
return r.markBatchImageJobSettledWithSQL(ctx, r.sql, params)
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
if err := r.markBatchImageJobSettledWithSQL(ctx, tx, params); err != nil {
return err
}
return tx.Commit()
}
func (r *batchImageRepository) markBatchImageJobSettledWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.MarkBatchImageJobSettledParams) error {
now := time.Now()
if params.Now != nil {
now = *params.Now
}
outputExpiresAt := params.OutputExpiresAt
res, err := sqlq.ExecContext(ctx, `
UPDATE batch_image_jobs
SET status = 'completed',
actual_cost = $2,
manifest_hash = $3,
settled_at = CASE WHEN settled_at IS NULL THEN $4 ELSE settled_at END,
finished_at = CASE WHEN finished_at IS NULL THEN $4 ELSE finished_at END,
output_expires_at = CASE WHEN output_expires_at IS NULL THEN $5 ELSE output_expires_at END,
updated_at = $4,
version = version + 1
WHERE batch_id = $1
AND status = 'settling'
AND (manifest_hash IS NULL OR manifest_hash = '' OR manifest_hash = $3)`, params.BatchID, params.ActualCost, params.ManifestHash, now, outputExpiresAt)
if err != nil {
return err
}
affected, err := res.RowsAffected()
if err != nil {
return err
}
if affected == 0 {
job, getErr := scanBatchImageJob(sqlq.QueryRowContext(ctx, batchImageJobSelectSQL+" WHERE batch_id = $1", params.BatchID))
if getErr != nil {
return translatePersistenceError(getErr, service.ErrBatchImageJobNotFound, nil)
}
if job.Status != service.BatchImageJobStatusSettling {
if job.Status == service.BatchImageJobStatusCompleted {
return service.ErrBatchImageAlreadySettled
}
return service.ErrBatchImageSettlementInvalidStatus
}
return service.ErrBatchImageSettlementManifestConflict
}
return appendBatchImageEventWithSQL(ctx, sqlq, params.BatchID, "settlement_completed", params.EventPayload)
}
func (r *batchImageRepository) SetBatchImageJobSettlementFailed(ctx context.Context, batchID, code, message string) error {
_, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET last_error_code = $2,
last_error_message = $3,
retry_count = retry_count + 1,
updated_at = $4
WHERE batch_id = $1`, batchID, code, message, time.Now())
if err != nil {
return err
}
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "settlement_failed", map[string]any{
"error_code": code,
})
}
func (r *batchImageRepository) transitionBatchImageJobStatusWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID, toStatus string, opts service.BatchImageTransitionOptions) error {
var current string
if err := sqlq.QueryRowContext(ctx, `SELECT status FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, batchID).Scan(&current); err != nil {
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
if !service.CanTransitionBatchImageJob(current, toStatus) {
return service.ErrBatchImageInvalidTransition
}
now := time.Now()
if opts.Now != nil {
now = *opts.Now
}
if _, err := sqlq.ExecContext(ctx, `
UPDATE batch_image_jobs
SET
status = $2,
version = version + 1,
updated_at = $3,
last_error_code = CASE WHEN $2 = 'failed' THEN $4 ELSE last_error_code END,
last_error_message = CASE WHEN $2 = 'failed' THEN $5 ELSE last_error_message END,
submitted_at = CASE WHEN $2 = 'submitted' AND submitted_at IS NULL THEN $3 ELSE submitted_at END,
started_at = CASE WHEN $2 = 'running' AND started_at IS NULL THEN $3 ELSE started_at END,
finished_at = CASE WHEN $2 IN ('completed', 'failed', 'cancelled') AND finished_at IS NULL THEN $3 ELSE finished_at END,
settled_at = CASE WHEN $2 = 'completed' AND settled_at IS NULL THEN $3 ELSE settled_at END,
output_deleted_at = CASE WHEN $2 = 'output_deleted' AND output_deleted_at IS NULL THEN $3 ELSE output_deleted_at END
WHERE batch_id = $1`, batchID, toStatus, now, opts.ErrorCode, opts.ErrorMessage); err != nil {
return err
}
if opts.EventType != "" {
return appendBatchImageEventWithSQL(ctx, sqlq, batchID, opts.EventType, opts.EventPayload)
}
return nil
}
func (r *batchImageRepository) CreateBatchImageItem(ctx context.Context, params service.CreateBatchImageItemParams) (*service.BatchImageItem, error) {
item, err := createBatchImageItemWithSQL(ctx, r.sql, params)
if err != nil {
return nil, translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
}
return item, nil
}
func (r *batchImageRepository) BulkCreateBatchImageItems(ctx context.Context, params []service.CreateBatchImageItemParams) error {
if len(params) == 0 {
return nil
}
if r.db == nil {
for _, param := range params {
if _, err := createBatchImageItemWithSQL(ctx, r.sql, param); err != nil {
return translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
}
}
return nil
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
for _, param := range params {
if _, err := createBatchImageItemWithSQL(ctx, tx, param); err != nil {
return translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
}
}
return tx.Commit()
}
func (r *batchImageRepository) ReplaceBatchImageItemsForJob(ctx context.Context, batchID string, items []service.CreateBatchImageItemParams, counts service.BatchImageCounts) error {
if r.db == nil {
return r.replaceBatchImageItemsForJobWithSQL(ctx, r.sql, batchID, items, counts)
}
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
if err := r.replaceBatchImageItemsForJobWithSQL(ctx, tx, batchID, items, counts); err != nil {
return err
}
return tx.Commit()
}
func (r *batchImageRepository) replaceBatchImageItemsForJobWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID string, items []service.CreateBatchImageItemParams, counts service.BatchImageCounts) error {
var id int64
if err := sqlq.QueryRowContext(ctx, `SELECT id FROM batch_image_jobs WHERE batch_id = $1 FOR UPDATE`, batchID).Scan(&id); err != nil {
return translatePersistenceError(err, service.ErrBatchImageJobNotFound, nil)
}
if _, err := sqlq.ExecContext(ctx, `DELETE FROM batch_image_items WHERE job_id = $1`, batchID); err != nil {
return err
}
for _, item := range items {
item.JobID = batchID
if _, err := createBatchImageItemWithSQL(ctx, sqlq, item); err != nil {
return translatePersistenceError(err, nil, service.ErrBatchImageItemExists)
}
}
_, err := sqlq.ExecContext(ctx, `
UPDATE batch_image_jobs
SET success_count = $2,
fail_count = $3,
updated_at = $4
WHERE batch_id = $1`, batchID, counts.SuccessCount, counts.FailCount, time.Now())
return err
}
func (r *batchImageRepository) ListBatchImageItems(ctx context.Context, batchID string, filter service.BatchImageItemFilter) ([]*service.BatchImageItem, error) {
limit := filter.Limit
if limit <= 0 || limit > 500 {
limit = 100
}
if filter.Offset < 0 {
filter.Offset = 0
}
query := batchImageItemSelectSQL + " WHERE job_id = $1"
args := []any{batchID}
if filter.Status != "" {
query += " AND status = $2"
args = append(args, filter.Status)
}
query += " ORDER BY id ASC LIMIT $" + strconv.Itoa(len(args)+1) + " OFFSET $" + strconv.Itoa(len(args)+2)
args = append(args, limit, filter.Offset)
rows, err := r.sql.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var items []*service.BatchImageItem
for rows.Next() {
item, err := scanBatchImageItem(rows)
if err != nil {
return nil, err
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
func (r *batchImageRepository) ListBatchImageItemsForOwner(ctx context.Context, userID, apiKeyID int64, batchID string, filter service.BatchImageItemFilter) ([]*service.BatchImageItem, error) {
if _, err := r.GetBatchImageJobByBatchIDForOwner(ctx, userID, apiKeyID, batchID); err != nil {
return nil, err
}
return r.ListBatchImageItems(ctx, batchID, filter)
}
func (r *batchImageRepository) GetBatchImageJobForDownload(ctx context.Context, userID, apiKeyID int64, batchID string) (*service.BatchImageJob, error) {
return r.GetBatchImageJobByBatchIDForOwner(ctx, userID, apiKeyID, batchID)
}
func (r *batchImageRepository) GetBatchImageItemForDownload(ctx context.Context, batchID, customID string) (*service.BatchImageItem, error) {
item, err := scanBatchImageItem(r.sql.QueryRowContext(ctx, batchImageItemSelectSQL+`
WHERE job_id = $1 AND custom_id = $2`, batchID, customID))
if err != nil {
return nil, translatePersistenceError(err, service.ErrBatchImageItemNotFound, nil)
}
return item, nil
}
func (r *batchImageRepository) ListBatchImageItemsForDownload(ctx context.Context, batchID string, status string, limit int) ([]*service.BatchImageItem, error) {
return r.ListBatchImageItems(ctx, batchID, service.BatchImageItemFilter{Status: status, Limit: limit})
}
func (r *batchImageRepository) ListBatchImageJobsDueForInputCleanup(ctx context.Context, cutoff time.Time, limit int) ([]*service.BatchImageJob, error) {
if limit <= 0 || limit > 1000 {
limit = 100
}
rows, err := r.sql.QueryContext(ctx, batchImageJobSelectSQL+`
WHERE input_deleted_at IS NULL
AND provider_input_ref IS NOT NULL
AND status IN ('completed', 'failed', 'cancelled', 'output_deleted')
AND COALESCE(finished_at, settled_at, updated_at, created_at) <= $1
ORDER BY id ASC
LIMIT $2`, cutoff, limit)
if err != nil {
return nil, err
}
defer rows.Close()
return scanBatchImageJobs(rows)
}
func (r *batchImageRepository) ListBatchImageJobsDueForOutputCleanup(ctx context.Context, now time.Time, limit int) ([]*service.BatchImageJob, error) {
if limit <= 0 || limit > 1000 {
limit = 100
}
rows, err := r.sql.QueryContext(ctx, batchImageJobSelectSQL+`
WHERE output_deleted_at IS NULL
AND provider_output_ref IS NOT NULL
AND status = 'completed'
AND output_expires_at IS NOT NULL
AND output_expires_at <= $1
ORDER BY output_expires_at ASC, id ASC
LIMIT $2`, now, limit)
if err != nil {
return nil, err
}
defer rows.Close()
return scanBatchImageJobs(rows)
}
func (r *batchImageRepository) MarkBatchImageInputDeleted(ctx context.Context, batchID string, deletedAt time.Time) error {
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET input_deleted_at = CASE WHEN input_deleted_at IS NULL THEN $2 ELSE input_deleted_at END,
updated_at = $2,
version = version + 1
WHERE batch_id = $1`, batchID, deletedAt)
if err != nil {
return err
}
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
return service.ErrBatchImageJobNotFound
}
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "input_cleanup_completed", map[string]any{
"batch_id": batchID,
"cleanup_target": "input",
"deleted_at": deletedAt.UTC().Format(time.RFC3339),
})
}
func (r *batchImageRepository) MarkBatchImageOutputDeleted(ctx context.Context, batchID string, deletedAt time.Time) error {
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET status = CASE WHEN status = 'completed' THEN 'output_deleted' ELSE status END,
output_deleted_at = CASE WHEN output_deleted_at IS NULL THEN $2 ELSE output_deleted_at END,
finished_at = CASE WHEN status = 'completed' AND finished_at IS NULL THEN $2 ELSE finished_at END,
updated_at = $2,
version = version + 1
WHERE batch_id = $1`, batchID, deletedAt)
if err != nil {
return err
}
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
return service.ErrBatchImageJobNotFound
}
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "output_cleanup_completed", map[string]any{
"batch_id": batchID,
"cleanup_target": "output",
"deleted_at": deletedAt.UTC().Format(time.RFC3339),
})
}
func (r *batchImageRepository) SetBatchImageOutputExpiresAt(ctx context.Context, batchID string, expiresAt time.Time) error {
res, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET output_expires_at = CASE WHEN output_expires_at IS NULL THEN $2 ELSE output_expires_at END,
updated_at = $3
WHERE batch_id = $1`, batchID, expiresAt, time.Now())
if err != nil {
return err
}
if affected, err := res.RowsAffected(); err == nil && affected == 0 {
return service.ErrBatchImageJobNotFound
}
return nil
}
func (r *batchImageRepository) RecordBatchImageCleanupFailure(ctx context.Context, batchID, code, message string) error {
_, err := r.sql.ExecContext(ctx, `
UPDATE batch_image_jobs
SET last_error_code = $2,
last_error_message = $3,
retry_count = retry_count + 1,
updated_at = $4
WHERE batch_id = $1`, batchID, code, message, time.Now())
if err != nil {
return err
}
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, "output_cleanup_failed", map[string]any{"error_code": code})
}
func (r *batchImageRepository) AppendBatchImageEvent(ctx context.Context, batchID, eventType string, payload any) error {
return appendBatchImageEventWithSQL(ctx, r.sql, batchID, eventType, payload)
}
func createBatchImageJobWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.CreateBatchImageJobParams) (*service.BatchImageJob, error) {
return scanBatchImageJob(sqlq.QueryRowContext(ctx, `
INSERT INTO batch_image_jobs (
batch_id, user_id, api_key_id, account_id, provider, model, status,
provider_job_name, provider_input_ref, provider_output_ref, gcs_input_uri, gcs_output_uri,
item_count, success_count, fail_count, cancelled_count,
estimated_cost, hold_amount, actual_cost, currency, hold_id,
idempotency_key, request_hash, manifest_hash, retry_count, output_expires_at
) VALUES (
$1, $2, $3, $4, $5, $6, $7,
$8, $9, $10, $11, $12,
$13, $14, $15, $16,
$17, $18, $19, $20, $21,
$22, $23, $24, $25, $26
)
RETURNING `+batchImageJobColumns,
params.BatchID, params.UserID, params.APIKeyID, params.AccountID, params.Provider, params.Model, params.Status,
params.ProviderJobName, params.ProviderInputRef, params.ProviderOutputRef, params.GCSInputURI, params.GCSOutputURI,
params.ItemCount, params.SuccessCount, params.FailCount, params.CancelledCount,
params.EstimatedCost, params.HoldAmount, params.ActualCost, params.Currency, params.HoldID,
params.IdempotencyKey, params.RequestHash, params.ManifestHash, params.RetryCount, params.OutputExpiresAt,
))
}
func createBatchImageItemWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, params service.CreateBatchImageItemParams) (*service.BatchImageItem, error) {
return scanBatchImageItem(sqlq.QueryRowContext(ctx, `
INSERT INTO batch_image_items (
job_id, custom_id, status, request_hash, prompt_preview, provider_source_object,
source_line_number, source_byte_offset, source_byte_length,
mime_type, file_extension, image_count,
error_code, error_message, billed_amount, indexed_at
) VALUES (
$1, $2, $3, $4, $5, $6,
$7, $8, $9,
$10, $11, $12,
$13, $14, $15, $16
)
RETURNING `+batchImageItemColumns,
params.JobID, params.CustomID, params.Status, params.RequestHash, params.PromptPreview, params.ProviderSourceObject,
params.SourceLineNumber, params.SourceByteOffset, params.SourceByteLength,
params.MimeType, params.FileExtension, params.ImageCount,
params.ErrorCode, params.ErrorMessage, params.BilledAmount, params.IndexedAt,
))
}
func appendBatchImageEventWithSQL(ctx context.Context, sqlq batchImageSQLExecutor, batchID, eventType string, payload any) error {
var payloadArg any
if payload != nil {
payloadBytes, err := json.Marshal(payload)
if err != nil {
return err
}
payloadArg = string(payloadBytes)
}
_, err := sqlq.ExecContext(ctx, `
INSERT INTO batch_image_events (job_id, event_type, payload)
VALUES ($1, $2, $3)`, batchID, eventType, payloadArg)
return err
}
type rowScanner interface {
Scan(dest ...any) error
}
const batchImageJobColumns = `
id, batch_id, user_id, api_key_id, account_id, provider, model, status,
provider_job_name, provider_input_ref, provider_output_ref, gcs_input_uri, gcs_output_uri,
item_count, success_count, fail_count, cancelled_count,
estimated_cost, hold_amount, actual_cost, currency, hold_id,
idempotency_key, request_hash, manifest_hash,
retry_count, version, output_expires_at, input_deleted_at, output_deleted_at,
last_error_code, last_error_message,
created_at, updated_at, submitted_at, started_at, finished_at, settled_at`
const batchImageJobSelectSQL = `SELECT ` + batchImageJobColumns + ` FROM batch_image_jobs`
func scanBatchImageJob(row rowScanner) (*service.BatchImageJob, error) {
var job service.BatchImageJob
var apiKeyID, accountID sql.NullInt64
var providerJobName, providerInputRef, providerOutputRef, gcsInputURI, gcsOutputURI sql.NullString
var holdAmount, actualCost sql.NullFloat64
var holdID, idempotencyKey, requestHash, manifestHash sql.NullString
var outputExpiresAt, inputDeletedAt, outputDeletedAt sql.NullTime
var lastErrorCode, lastErrorMessage sql.NullString
var submittedAt, startedAt, finishedAt, settledAt sql.NullTime
err := row.Scan(
&job.ID, &job.BatchID, &job.UserID, &apiKeyID, &accountID, &job.Provider, &job.Model, &job.Status,
&providerJobName, &providerInputRef, &providerOutputRef, &gcsInputURI, &gcsOutputURI,
&job.ItemCount, &job.SuccessCount, &job.FailCount, &job.CancelledCount,
&job.EstimatedCost, &holdAmount, &actualCost, &job.Currency, &holdID,
&idempotencyKey, &requestHash, &manifestHash,
&job.RetryCount, &job.Version, &outputExpiresAt, &inputDeletedAt, &outputDeletedAt,
&lastErrorCode, &lastErrorMessage,
&job.CreatedAt, &job.UpdatedAt, &submittedAt, &startedAt, &finishedAt, &settledAt,
)
if err != nil {
return nil, err
}
job.APIKeyID = batchImageNullInt64Ptr(apiKeyID)
job.AccountID = batchImageNullInt64Ptr(accountID)
job.ProviderJobName = batchImageNullStringPtr(providerJobName)
job.ProviderInputRef = batchImageNullStringPtr(providerInputRef)
job.ProviderOutputRef = batchImageNullStringPtr(providerOutputRef)
job.GCSInputURI = batchImageNullStringPtr(gcsInputURI)
job.GCSOutputURI = batchImageNullStringPtr(gcsOutputURI)
job.HoldAmount = batchImageNullFloat64Ptr(holdAmount)
job.ActualCost = batchImageNullFloat64Ptr(actualCost)
job.HoldID = batchImageNullStringPtr(holdID)
job.IdempotencyKey = batchImageNullStringPtr(idempotencyKey)
job.RequestHash = batchImageNullStringPtr(requestHash)
job.ManifestHash = batchImageNullStringPtr(manifestHash)
job.OutputExpiresAt = batchImageNullTimePtr(outputExpiresAt)
job.InputDeletedAt = batchImageNullTimePtr(inputDeletedAt)
job.OutputDeletedAt = batchImageNullTimePtr(outputDeletedAt)
job.LastErrorCode = batchImageNullStringPtr(lastErrorCode)
job.LastErrorMessage = batchImageNullStringPtr(lastErrorMessage)
job.SubmittedAt = batchImageNullTimePtr(submittedAt)
job.StartedAt = batchImageNullTimePtr(startedAt)
job.FinishedAt = batchImageNullTimePtr(finishedAt)
job.SettledAt = batchImageNullTimePtr(settledAt)
return &job, nil
}
func scanBatchImageJobs(rows *sql.Rows) ([]*service.BatchImageJob, error) {
var jobs []*service.BatchImageJob
for rows.Next() {
job, err := scanBatchImageJob(rows)
if err != nil {
return nil, err
}
jobs = append(jobs, job)
}
if err := rows.Err(); err != nil {
return nil, err
}
return jobs, nil
}
const batchImageItemColumns = `
id, job_id, custom_id, status, request_hash, prompt_preview, provider_source_object,
source_line_number, source_byte_offset, source_byte_length,
mime_type, file_extension, image_count,
error_code, error_message, billed_amount,
created_at, indexed_at`
const batchImageItemSelectSQL = `SELECT ` + batchImageItemColumns + ` FROM batch_image_items`
func scanBatchImageItem(row rowScanner) (*service.BatchImageItem, error) {
var item service.BatchImageItem
var requestHash, promptPreview, providerSourceObject sql.NullString
var sourceLineNumber sql.NullInt64
var sourceByteOffset, sourceByteLength sql.NullInt64
var mimeType, fileExtension, errorCode, errorMessage sql.NullString
var billedAmount sql.NullFloat64
var indexedAt sql.NullTime
err := row.Scan(
&item.ID, &item.JobID, &item.CustomID, &item.Status, &requestHash, &promptPreview, &providerSourceObject,
&sourceLineNumber, &sourceByteOffset, &sourceByteLength,
&mimeType, &fileExtension, &item.ImageCount,
&errorCode, &errorMessage, &billedAmount,
&item.CreatedAt, &indexedAt,
)
if err != nil {
return nil, err
}
item.RequestHash = batchImageNullStringPtr(requestHash)
item.PromptPreview = batchImageNullStringPtr(promptPreview)
item.ProviderSourceObject = batchImageNullStringPtr(providerSourceObject)
item.SourceLineNumber = batchImageNullIntPtr(sourceLineNumber)
item.SourceByteOffset = batchImageNullInt64Ptr(sourceByteOffset)
item.SourceByteLength = batchImageNullInt64Ptr(sourceByteLength)
item.MimeType = batchImageNullStringPtr(mimeType)
item.FileExtension = batchImageNullStringPtr(fileExtension)
item.ErrorCode = batchImageNullStringPtr(errorCode)
item.ErrorMessage = batchImageNullStringPtr(errorMessage)
item.BilledAmount = batchImageNullFloat64Ptr(billedAmount)
item.IndexedAt = batchImageNullTimePtr(indexedAt)
return &item, nil
}
func batchImageNullStringPtr(v sql.NullString) *string {
if !v.Valid {
return nil
}
return &v.String
}
func batchImageNullInt64Ptr(v sql.NullInt64) *int64 {
if !v.Valid {
return nil
}
return &v.Int64
}
func batchImageNullIntPtr(v sql.NullInt64) *int {
if !v.Valid {
return nil
}
i := int(v.Int64)
return &i
}
func batchImageNullFloat64Ptr(v sql.NullFloat64) *float64 {
if !v.Valid {
return nil
}
return &v.Float64
}
func batchImageNullTimePtr(v sql.NullTime) *time.Time {
if !v.Valid {
return nil
}
return &v.Time
}
var _ service.BatchImageRepository = (*batchImageRepository)(nil)
@@ -0,0 +1,339 @@
//go:build integration
package repository
import (
"context"
"errors"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/stretchr/testify/require"
)
func TestBatchImageRepository_CreateJobAndDuplicates(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "create")
job, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-2.5-flash-image",
ItemCount: 2,
EstimatedCost: 0.02,
})
require.NoError(t, err)
require.Equal(t, batchID, job.BatchID)
require.Equal(t, service.BatchImageJobStatusCreated, job.Status)
require.Equal(t, "USD", job.Currency)
_, err = repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-2.5-flash-image",
ItemCount: 1,
})
require.Error(t, err)
require.True(t, errors.Is(err, service.ErrBatchImageJobExists))
}
func TestBatchImageRepository_InvalidProvider(t *testing.T) {
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
_, err := repo.CreateBatchImageJob(context.Background(), service.CreateBatchImageJobParams{
BatchID: batchImageTestID(t, "provider"),
UserID: 1001,
Provider: "unknown",
Model: "gemini-2.5-flash-image",
ItemCount: 1,
})
require.Error(t, err)
require.True(t, errors.Is(err, service.ErrBatchImageInvalidProvider))
}
func TestBatchImageRepository_TransitionIncrementsVersionAndEvents(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "transition")
now := time.Date(2026, 7, 3, 8, 0, 0, 0, time.UTC)
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderVertex,
Model: "gemini-2.5-flash-image",
ItemCount: 1,
})
require.NoError(t, err)
err = repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusUploading, service.BatchImageTransitionOptions{
EventType: "status_changed",
EventPayload: map[string]any{"to": service.BatchImageJobStatusUploading},
Now: &now,
})
require.NoError(t, err)
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
require.NoError(t, err)
require.Equal(t, service.BatchImageJobStatusUploading, job.Status)
require.Equal(t, 1, job.Version)
var eventCount int
err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM batch_image_events WHERE job_id = $1 AND event_type = 'status_changed'`, batchID).Scan(&eventCount)
require.NoError(t, err)
require.Equal(t, 1, eventCount)
}
func TestBatchImageRepository_InvalidTransition(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "invalid-transition")
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-2.5-flash-image",
ItemCount: 1,
})
require.NoError(t, err)
err = repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusRunning, service.BatchImageTransitionOptions{})
require.Error(t, err)
require.True(t, errors.Is(err, service.ErrBatchImageInvalidTransition))
}
func TestBatchImageRepository_TerminalStatusCannotMoveBack(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "terminal")
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-2.5-flash-image",
Status: service.BatchImageJobStatusCompleted,
ItemCount: 1,
})
require.NoError(t, err)
err = repo.TransitionBatchImageJobStatus(ctx, batchID, service.BatchImageJobStatusRunning, service.BatchImageTransitionOptions{})
require.Error(t, err)
require.True(t, errors.Is(err, service.ErrBatchImageInvalidTransition))
}
func TestBatchImageRepository_ItemCustomIDUniqueness(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
firstBatchID := batchImageTestID(t, "items-a")
secondBatchID := batchImageTestID(t, "items-b")
for _, batchID := range []string{firstBatchID, secondBatchID} {
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-2.5-flash-image",
ItemCount: 1,
})
require.NoError(t, err)
}
_, err := repo.CreateBatchImageItem(ctx, service.CreateBatchImageItemParams{
JobID: firstBatchID,
CustomID: "line-1",
Status: service.BatchImageItemStatusSuccess,
ImageCount: 1,
})
require.NoError(t, err)
_, err = repo.CreateBatchImageItem(ctx, service.CreateBatchImageItemParams{
JobID: firstBatchID,
CustomID: "line-1",
Status: service.BatchImageItemStatusFailed,
})
require.Error(t, err)
require.True(t, errors.Is(err, service.ErrBatchImageItemExists))
_, err = repo.CreateBatchImageItem(ctx, service.CreateBatchImageItemParams{
JobID: secondBatchID,
CustomID: "line-1",
Status: service.BatchImageItemStatusSuccess,
ImageCount: 1,
})
require.NoError(t, err)
items, err := repo.ListBatchImageItems(ctx, firstBatchID, service.BatchImageItemFilter{})
require.NoError(t, err)
require.Len(t, items, 1)
}
func TestBatchImageRepository_ReplaceBatchImageItemsForJob(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "replace-items")
lineOne := 1
lineTwo := 2
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-2.5-flash-image",
ItemCount: 2,
})
require.NoError(t, err)
err = repo.ReplaceBatchImageItemsForJob(ctx, batchID, []service.CreateBatchImageItemParams{
{CustomID: "old", Status: service.BatchImageItemStatusSuccess, SourceLineNumber: &lineOne, ImageCount: 1},
}, service.BatchImageCounts{SuccessCount: 1})
require.NoError(t, err)
err = repo.ReplaceBatchImageItemsForJob(ctx, batchID, []service.CreateBatchImageItemParams{
{CustomID: "new-ok", Status: service.BatchImageItemStatusSuccess, SourceLineNumber: &lineOne, ImageCount: 1},
{CustomID: "new-fail", Status: service.BatchImageItemStatusFailed, SourceLineNumber: &lineTwo, ErrorCode: batchImageTestStringPtr("SAFETY_BLOCKED")},
}, service.BatchImageCounts{SuccessCount: 1, FailCount: 1})
require.NoError(t, err)
items, err := repo.ListBatchImageItems(ctx, batchID, service.BatchImageItemFilter{})
require.NoError(t, err)
require.Len(t, items, 2)
require.Equal(t, "new-ok", items[0].CustomID)
require.Equal(t, "new-fail", items[1].CustomID)
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
require.NoError(t, err)
require.Equal(t, 1, job.SuccessCount)
require.Equal(t, 1, job.FailCount)
}
func TestBatchImageRepository_MarkBatchImageJobSettled(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "settled")
apiKeyID := int64(2001)
accountID := int64(3001)
providerJob := "providers/job"
outputRef := "files/output"
now := time.Date(2026, 7, 4, 10, 0, 0, 0, time.UTC)
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
APIKeyID: &apiKeyID,
AccountID: &accountID,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-image",
Status: service.BatchImageJobStatusSettling,
ProviderJobName: &providerJob,
ProviderOutputRef: &outputRef,
ItemCount: 3,
SuccessCount: 2,
FailCount: 1,
})
require.NoError(t, err)
err = repo.MarkBatchImageJobSettled(ctx, service.MarkBatchImageJobSettledParams{
BatchID: batchID,
ActualCost: 0.5,
ManifestHash: "manifest-hash",
EventPayload: map[string]any{"request_id": "batch_image_settlement:" + batchID},
Now: &now,
})
require.NoError(t, err)
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
require.NoError(t, err)
require.Equal(t, service.BatchImageJobStatusCompleted, job.Status)
require.NotNil(t, job.ActualCost)
require.Equal(t, 0.5, *job.ActualCost)
require.Equal(t, "manifest-hash", batchImageDerefTest(job.ManifestHash))
require.NotNil(t, job.SettledAt)
require.Equal(t, now, *job.SettledAt)
var eventCount int
err = tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM batch_image_events WHERE job_id = $1 AND event_type = 'settlement_completed'`, batchID).Scan(&eventCount)
require.NoError(t, err)
require.Equal(t, 1, eventCount)
}
func TestBatchImageRepository_SetBatchImageJobSettlementFailed(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "settlement-failed")
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderGeminiAPI,
Model: "gemini-image",
Status: service.BatchImageJobStatusSettling,
ItemCount: 1,
SuccessCount: 1,
})
require.NoError(t, err)
err = repo.SetBatchImageJobSettlementFailed(ctx, batchID, "SETTLEMENT_BILLING_FAILED", "temporary")
require.NoError(t, err)
job, err := repo.GetBatchImageJobByBatchID(ctx, batchID)
require.NoError(t, err)
require.Equal(t, service.BatchImageJobStatusSettling, job.Status)
require.Equal(t, "SETTLEMENT_BILLING_FAILED", batchImageDerefTest(job.LastErrorCode))
require.Equal(t, "temporary", batchImageDerefTest(job.LastErrorMessage))
require.Equal(t, 1, job.RetryCount)
}
func TestBatchImageRepository_AppendEvent(t *testing.T) {
ctx := context.Background()
tx := testTx(t)
repo := newBatchImageRepositoryWithSQL(tx)
batchID := batchImageTestID(t, "event")
_, err := repo.CreateBatchImageJob(ctx, service.CreateBatchImageJobParams{
BatchID: batchID,
UserID: 1001,
Provider: service.BatchImageProviderVertex,
Model: "gemini-2.5-flash-image",
ItemCount: 1,
})
require.NoError(t, err)
err = repo.AppendBatchImageEvent(ctx, batchID, "job_created", map[string]any{"batch_id": batchID})
require.NoError(t, err)
var payload string
err = tx.QueryRowContext(ctx, `SELECT payload::text FROM batch_image_events WHERE job_id = $1 AND event_type = 'job_created'`, batchID).Scan(&payload)
require.NoError(t, err)
require.Contains(t, payload, batchID)
}
func batchImageTestID(t *testing.T, prefix string) string {
t.Helper()
return "imgbatch_" + uniqueTestValue(t, prefix)
}
func batchImageTestStringPtr(v string) *string {
return &v
}
func batchImageDerefTest(v *string) string {
if v == nil {
return ""
}
return *v
}
+3
View File
@@ -77,6 +77,7 @@ var ProviderSet = wire.NewSet(
NewAnnouncementReadRepository,
NewUsageLogRepository,
NewUsageBillingRepository,
NewBatchImageRepository,
NewIdempotencyRepository,
NewUsageCleanupRepository,
NewDashboardAggregationRepository,
@@ -115,6 +116,8 @@ var ProviderSet = wire.NewSet(
NewRedeemCache,
NewUpdateCache,
NewGeminiTokenCache,
NewBatchImageQueue,
NewBatchImageDownloadLimiter,
NewLeaderLockCache,
ProvideSchedulerCache,
NewSchedulerOutboxRepository,