mirror of
https://github.com/restic/restic.git
synced 2026-11-19 23:01:08 +00:00
repository: inline MasterIndex interface into Repository interface
This commit is contained in:
1 parent
3eeb6723cd
commit
4df887406f
22 files changed
+109
-94
No files matched your search
@@ -158,11 +158,10 @@ func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs []re
|
||||
errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize))
|
||||
}
|
||||
|
||||
idx := r.Index()
|
||||
for _, blob := range blobs {
|
||||
// Check if blob is contained in index and position is correct
|
||||
idxHas := false
|
||||
for _, pb := range idx.Lookup(blob.BlobHandle) {
|
||||
for _, pb := range r.LookupBlob(blob.BlobHandle) {
|
||||
if pb.PackID == id && pb.Blob == blob {
|
||||
idxHas = true
|
||||
break
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"sort"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/index"
|
||||
"github.com/restic/restic/internal/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
@@ -67,7 +66,7 @@ type PrunePlan struct {
|
||||
removePacks restic.IDSet // packs to remove
|
||||
ignorePacks restic.IDSet // packs to ignore when rebuilding the index
|
||||
|
||||
repo restic.Repository
|
||||
repo *Repository
|
||||
stats PruneStats
|
||||
opts PruneOptions
|
||||
}
|
||||
@@ -89,7 +88,7 @@ type packInfoWithID struct {
|
||||
|
||||
// PlanPrune selects which files to rewrite and which to delete and which blobs to keep.
|
||||
// Also some summary statistics are returned.
|
||||
func PlanPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, getUsedBlobs func(ctx context.Context, repo restic.Repository) (usedBlobs restic.CountedBlobSet, err error), printer progress.Printer) (*PrunePlan, error) {
|
||||
func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsedBlobs func(ctx context.Context, repo restic.Repository) (usedBlobs restic.CountedBlobSet, err error), printer progress.Printer) (*PrunePlan, error) {
|
||||
var stats PruneStats
|
||||
|
||||
if opts.UnsafeRecovery {
|
||||
@@ -109,7 +108,7 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, g
|
||||
}
|
||||
|
||||
printer.P("searching used packs...\n")
|
||||
keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats, printer)
|
||||
keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo, usedBlobs, &stats, printer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -124,7 +123,7 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, g
|
||||
blobCount := keepBlobs.Len()
|
||||
// when repacking, we do not want to keep blobs which are
|
||||
// already contained in kept packs, so delete them from keepBlobs
|
||||
err := repo.Index().Each(ctx, func(blob restic.PackedBlob) {
|
||||
err := repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
|
||||
if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
|
||||
return
|
||||
}
|
||||
@@ -151,11 +150,11 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, g
|
||||
return &plan, nil
|
||||
}
|
||||
|
||||
func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *PruneStats, printer progress.Printer) (restic.CountedBlobSet, map[restic.ID]packInfo, error) {
|
||||
func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs restic.CountedBlobSet, stats *PruneStats, printer progress.Printer) (restic.CountedBlobSet, map[restic.ID]packInfo, error) {
|
||||
// iterate over all blobs in index to find out which blobs are duplicates
|
||||
// The counter in usedBlobs describes how many instances of the blob exist in the repository index
|
||||
// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
|
||||
err := idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
err := idx.ListBlobs(ctx, func(blob restic.PackedBlob) {
|
||||
bh := blob.BlobHandle
|
||||
count, ok := usedBlobs[bh]
|
||||
if ok {
|
||||
@@ -205,7 +204,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
|
||||
|
||||
hasDuplicates := false
|
||||
// iterate over all blobs in index to generate packInfo
|
||||
err = idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) {
|
||||
ip := indexPack[blob.PackID]
|
||||
|
||||
// Set blob type if not yet set
|
||||
@@ -260,7 +259,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
|
||||
// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
|
||||
if hasDuplicates {
|
||||
// iterate again over all blobs in index (this is pretty cheap, all in-mem)
|
||||
err = idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) {
|
||||
bh := blob.BlobHandle
|
||||
count, ok := usedBlobs[bh]
|
||||
// skip non-duplicate, aka. normal blobs
|
||||
@@ -581,7 +580,7 @@ func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) (e
|
||||
|
||||
if plan.opts.UnsafeRecovery {
|
||||
printer.P("deleting index files\n")
|
||||
indexFiles := repo.Index().(*index.MasterIndex).IDs()
|
||||
indexFiles := repo.idx.IDs()
|
||||
err = deleteFiles(ctx, false, repo, indexFiles, restic.IndexFile, printer)
|
||||
if err != nil {
|
||||
return errors.Fatalf("%s", err)
|
||||
|
||||
@@ -54,7 +54,7 @@ func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito
|
||||
downloadQueue := make(chan restic.PackBlobs)
|
||||
wg.Go(func() error {
|
||||
defer close(downloadQueue)
|
||||
for pbs := range repo.Index().ListPacks(wgCtx, packs) {
|
||||
for pbs := range repo.ListPacksFromIndex(wgCtx, packs) {
|
||||
var packBlobs []restic.Blob
|
||||
keepMutex.Lock()
|
||||
// filter out unnecessary blobs
|
||||
|
||||
@@ -145,9 +145,8 @@ func listFiles(t *testing.T, repo restic.Lister, tpe backend.FileType) restic.ID
|
||||
func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSet) restic.IDSet {
|
||||
packs := restic.NewIDSet()
|
||||
|
||||
idx := repo.Index()
|
||||
for h := range blobs {
|
||||
list := idx.Lookup(h)
|
||||
list := repo.LookupBlob(h)
|
||||
if len(list) == 0 {
|
||||
t.Fatal("Failed to find blob", h.ID.Str(), "with type", h.Type)
|
||||
}
|
||||
@@ -195,7 +194,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) {
|
||||
})
|
||||
rtest.OK(t, err)
|
||||
|
||||
err = repo.Index().Save(context.TODO(), repo, restic.NewIDSet(), obsoleteIndexes, restic.MasterIndexSaveOpts{})
|
||||
err = repo.SaveIndex(context.TODO(), restic.NewIDSet(), obsoleteIndexes, restic.MasterIndexSaveOpts{})
|
||||
rtest.OK(t, err)
|
||||
}
|
||||
|
||||
@@ -252,10 +251,8 @@ func testRepack(t *testing.T, version uint) {
|
||||
}
|
||||
}
|
||||
|
||||
idx := repo.Index()
|
||||
|
||||
for h := range keepBlobs {
|
||||
list := idx.Lookup(h)
|
||||
list := repo.LookupBlob(h)
|
||||
if len(list) == 0 {
|
||||
t.Errorf("unable to find blob %v in repo", h.ID.Str())
|
||||
continue
|
||||
@@ -318,10 +315,8 @@ func testRepackCopy(t *testing.T, version uint) {
|
||||
rebuildIndex(t, dstRepo)
|
||||
reloadIndex(t, dstRepo)
|
||||
|
||||
idx := dstRepo.Index()
|
||||
|
||||
for h := range keepBlobs {
|
||||
list := idx.Lookup(h)
|
||||
list := dstRepo.LookupBlob(h)
|
||||
if len(list) == 0 {
|
||||
t.Errorf("unable to find blob %v in repo", h.ID.Str())
|
||||
continue
|
||||
|
||||
@@ -54,7 +54,7 @@ func RepairIndex(ctx context.Context, repo *Repository, opts RepairIndexOptions,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
packSizeFromIndex, err = pack.Size(ctx, repo.Index(), false)
|
||||
packSizeFromIndex, err = pack.Size(ctx, repo, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -115,7 +115,7 @@ func rebuildIndexFiles(ctx context.Context, repo restic.Repository, removePacks
|
||||
printer.P("rebuilding index\n")
|
||||
|
||||
bar := printer.NewCounter("packs processed")
|
||||
return repo.Index().Save(ctx, repo, removePacks, extraObsolete, restic.MasterIndexSaveOpts{
|
||||
return repo.SaveIndex(ctx, removePacks, extraObsolete, restic.MasterIndexSaveOpts{
|
||||
SaveProgress: bar,
|
||||
DeleteProgress: func() *progress.Counter {
|
||||
return printer.NewCounter("old indexes deleted")
|
||||
|
||||
@@ -21,7 +21,7 @@ func RepairPacks(ctx context.Context, repo restic.Repository, ids restic.IDSet,
|
||||
|
||||
wg.Go(func() error {
|
||||
// examine all data the indexes have for the pack file
|
||||
for b := range repo.Index().ListPacks(wgCtx, ids) {
|
||||
for b := range repo.ListPacksFromIndex(wgCtx, ids) {
|
||||
blobs := b.Blobs
|
||||
if len(blobs) == 0 {
|
||||
printer.E("no blobs found for pack %v", b.PackID)
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
|
||||
func listBlobs(repo restic.Repository) restic.BlobSet {
|
||||
blobs := restic.NewBlobSet()
|
||||
_ = repo.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
_ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
|
||||
blobs.Insert(pb.BlobHandle)
|
||||
})
|
||||
return blobs
|
||||
@@ -68,7 +68,7 @@ func testRepairBrokenPack(t *testing.T, version uint) {
|
||||
|
||||
// find blob that starts at offset 0
|
||||
var damagedBlob restic.BlobHandle
|
||||
for blobs := range repo.Index().ListPacks(context.TODO(), restic.NewIDSet(damagedID)) {
|
||||
for blobs := range repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(damagedID)) {
|
||||
for _, blob := range blobs.Blobs {
|
||||
if blob.Offset == 0 {
|
||||
damagedBlob = blob.BlobHandle
|
||||
@@ -91,7 +91,7 @@ func testRepairBrokenPack(t *testing.T, version uint) {
|
||||
|
||||
// all blobs in the file are broken
|
||||
damagedBlobs := restic.NewBlobSet()
|
||||
for blobs := range repo.Index().ListPacks(context.TODO(), restic.NewIDSet(damagedID)) {
|
||||
for blobs := range repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(damagedID)) {
|
||||
for _, blob := range blobs.Blobs {
|
||||
damagedBlobs.Insert(blob.BlobHandle)
|
||||
}
|
||||
|
||||
@@ -300,11 +300,6 @@ func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, bu
|
||||
return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].BlobHandle, len(blobs))
|
||||
}
|
||||
|
||||
// LookupBlobSize returns the size of blob id.
|
||||
func (r *Repository) LookupBlobSize(id restic.ID, tpe restic.BlobType) (uint, bool) {
|
||||
return r.idx.LookupSize(restic.BlobHandle{ID: id, Type: tpe})
|
||||
}
|
||||
|
||||
func (r *Repository) getZstdEncoder() *zstd.Encoder {
|
||||
r.allocEnc.Do(func() {
|
||||
level := zstd.SpeedDefault
|
||||
@@ -583,9 +578,31 @@ func (r *Repository) Connections() uint {
|
||||
return r.be.Connections()
|
||||
}
|
||||
|
||||
// Index returns the currently used MasterIndex.
|
||||
func (r *Repository) Index() restic.MasterIndex {
|
||||
return r.idx
|
||||
func (r *Repository) HasBlob(bh restic.BlobHandle) bool {
|
||||
return r.idx.Has(bh)
|
||||
}
|
||||
|
||||
func (r *Repository) LookupBlob(bh restic.BlobHandle) []restic.PackedBlob {
|
||||
return r.idx.Lookup(bh)
|
||||
}
|
||||
|
||||
// LookupBlobSize returns the size of blob id.
|
||||
func (r *Repository) LookupBlobSize(id restic.ID, tpe restic.BlobType) (uint, bool) {
|
||||
return r.idx.LookupSize(restic.BlobHandle{ID: id, Type: tpe})
|
||||
}
|
||||
|
||||
func (r *Repository) SaveIndex(ctx context.Context, excludePacks restic.IDSet, extraObsolete restic.IDs, opts restic.MasterIndexSaveOpts) error {
|
||||
return r.idx.Save(ctx, r, excludePacks, extraObsolete, opts)
|
||||
}
|
||||
|
||||
// ListBlobs runs fn on all blobs known to the index. When the context is cancelled,
|
||||
// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
|
||||
func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackedBlob)) error {
|
||||
return r.idx.Each(ctx, fn)
|
||||
}
|
||||
|
||||
func (r *Repository) ListPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs {
|
||||
return r.idx.ListPacks(ctx, packs)
|
||||
}
|
||||
|
||||
// SetIndex instructs the repository to use the given index.
|
||||
|
||||
@@ -161,7 +161,7 @@ func TestLoadBlobBroken(t *testing.T) {
|
||||
data, err := repo.LoadBlob(context.TODO(), restic.TreeBlob, id, nil)
|
||||
rtest.OK(t, err)
|
||||
rtest.Assert(t, bytes.Equal(buf, data), "data mismatch")
|
||||
pack := repo.Index().Lookup(restic.BlobHandle{Type: restic.TreeBlob, ID: id})[0].PackID
|
||||
pack := repo.LookupBlob(restic.BlobHandle{Type: restic.TreeBlob, ID: id})[0].PackID
|
||||
rtest.Assert(t, c.Has(backend.Handle{Type: restic.PackFile, Name: pack.String()}), "expected tree pack to be cached")
|
||||
}
|
||||
|
||||
@@ -439,7 +439,7 @@ func TestListPack(t *testing.T) {
|
||||
repo.UseCache(c)
|
||||
|
||||
// Forcibly cache pack file
|
||||
packID := repo.Index().Lookup(restic.BlobHandle{Type: restic.TreeBlob, ID: id})[0].PackID
|
||||
packID := repo.LookupBlob(restic.BlobHandle{Type: restic.TreeBlob, ID: id})[0].PackID
|
||||
rtest.OK(t, be.Load(context.TODO(), backend.Handle{Type: restic.PackFile, IsMetadata: true, Name: packID.String()}, 0, 0, func(rd io.Reader) error { return nil }))
|
||||
|
||||
// Get size to list pack
|
||||
|
||||
Reference in new issue
Block a user