mirror of
https://github.com/restic/restic.git
synced 2026-11-13 22:43:03 +00:00
let index.Each() and pack.Size() return error on canceled context
This forces a caller to actually check that the function did complete.
This commit is contained in:
1 parent
31624aeffd
commit
940a3159b5
20 files changed
+107
-72
No files matched your search
@@ -124,14 +124,14 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, g
|
||||
blobCount := keepBlobs.Len()
|
||||
// when repacking, we do not want to keep blobs which are
|
||||
// already contained in kept packs, so delete them from keepBlobs
|
||||
repo.Index().Each(ctx, func(blob restic.PackedBlob) {
|
||||
err := repo.Index().Each(ctx, func(blob restic.PackedBlob) {
|
||||
if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
|
||||
return
|
||||
}
|
||||
keepBlobs.Delete(blob.BlobHandle)
|
||||
})
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if keepBlobs.Len() < blobCount/2 {
|
||||
@@ -155,7 +155,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
|
||||
// iterate over all blobs in index to find out which blobs are duplicates
|
||||
// The counter in usedBlobs describes how many instances of the blob exist in the repository index
|
||||
// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
|
||||
idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
err := idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
bh := blob.BlobHandle
|
||||
count, ok := usedBlobs[bh]
|
||||
if ok {
|
||||
@@ -169,8 +169,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
|
||||
usedBlobs[bh] = count
|
||||
}
|
||||
})
|
||||
if ctx.Err() != nil {
|
||||
return nil, nil, ctx.Err()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Check if all used blobs have been found in index
|
||||
@@ -194,14 +194,18 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
|
||||
indexPack := make(map[restic.ID]packInfo)
|
||||
|
||||
// save computed pack header size
|
||||
for pid, hdrSize := range pack.Size(ctx, idx, true) {
|
||||
sz, err := pack.Size(ctx, idx, true)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for pid, hdrSize := range sz {
|
||||
// initialize tpe with NumBlobTypes to indicate it's not set
|
||||
indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)}
|
||||
}
|
||||
|
||||
hasDuplicates := false
|
||||
// iterate over all blobs in index to generate packInfo
|
||||
idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
err = idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
ip := indexPack[blob.PackID]
|
||||
|
||||
// Set blob type if not yet set
|
||||
@@ -246,8 +250,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
|
||||
// update indexPack
|
||||
indexPack[blob.PackID] = ip
|
||||
})
|
||||
if ctx.Err() != nil {
|
||||
return nil, nil, ctx.Err()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// if duplicate blobs exist, those will be set to either "used" or "unused":
|
||||
@@ -256,7 +260,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
|
||||
// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
|
||||
if hasDuplicates {
|
||||
// iterate again over all blobs in index (this is pretty cheap, all in-mem)
|
||||
idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
err = idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
bh := blob.BlobHandle
|
||||
count, ok := usedBlobs[bh]
|
||||
// skip non-duplicate, aka. normal blobs
|
||||
@@ -294,9 +298,9 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
|
||||
// update indexPack
|
||||
indexPack[blob.PackID] = ip
|
||||
})
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil, nil, ctx.Err()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Sanity check. If no duplicates exist, all blobs have value 1. After handling
|
||||
|
||||
@@ -54,7 +54,10 @@ func RepairIndex(ctx context.Context, repo *Repository, opts RepairIndexOptions,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
packSizeFromIndex = pack.Size(ctx, repo.Index(), false)
|
||||
packSizeFromIndex, err = pack.Size(ctx, repo.Index(), false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
printer.P("getting pack files to read...\n")
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
|
||||
func listBlobs(repo restic.Repository) restic.BlobSet {
|
||||
blobs := restic.NewBlobSet()
|
||||
repo.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
_ = repo.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
blobs.Insert(pb.BlobHandle)
|
||||
})
|
||||
return blobs
|
||||
|
||||
@@ -704,11 +704,14 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
|
||||
defer cancel()
|
||||
|
||||
invalidIndex := false
|
||||
r.idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
err := r.idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
if blob.IsCompressed() {
|
||||
invalidIndex = true
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if invalidIndex {
|
||||
return errors.New("index uses feature not supported by repository version 1")
|
||||
}
|
||||
|
||||
@@ -370,13 +370,13 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) {
|
||||
idx, err := loadIndex(context.TODO(), repo, id)
|
||||
rtest.OK(t, err)
|
||||
|
||||
idx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
rtest.OK(t, idx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
if _, ok := packEntries[pb.PackID]; !ok {
|
||||
packEntries[pb.PackID] = make(map[restic.ID]struct{})
|
||||
}
|
||||
|
||||
packEntries[pb.PackID][id] = struct{}{}
|
||||
})
|
||||
}))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user