mirror of
https://github.com/restic/restic.git
synced 2026-11-19 23:01:08 +00:00
backend: Improve Save()
As mentioned in issue [#1560](https://github.com/restic/restic/pull/1560#issuecomment-364689346) this changes the signature for `backend.Save()`. It now takes a parameter of interface type `RewindReader`, so that the backend implementations or our `RetryBackend` middleware can reset the reader to the beginning and then retry an upload operation. The `RewindReader` interface also provides a `Length()` method, which is used in the backend to get the size of the data to be saved. This removes several ugly hacks we had to do to pull the size back out of the `io.Reader` passed to `Save()` before. In the `s3` and `rest` backend this is actively used.
This commit is contained in:
1 parent
58306bfabb
commit
99f7fd74e3
29 files changed
+387
-204
No files matched your search
@@ -20,8 +20,8 @@ type Backend interface {
|
||||
// Close the backend
|
||||
Close() error
|
||||
|
||||
// Save stores the data in the backend under the given handle.
|
||||
Save(ctx context.Context, h Handle, rd io.Reader) error
|
||||
// Save stores the data from rd under the given handle.
|
||||
Save(ctx context.Context, h Handle, rd RewindReader) error
|
||||
|
||||
// Load runs fn with a reader that yields the contents of the file at h at the
|
||||
// given offset. If length is larger than zero, only a portion of the file
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package restic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
)
|
||||
|
||||
// RewindReader allows resetting the Reader to the beginning of the data.
|
||||
type RewindReader interface {
|
||||
io.Reader
|
||||
|
||||
// Rewind rewinds the reader so the same data can be read again from the
|
||||
// start.
|
||||
Rewind() error
|
||||
|
||||
// Length returns the number of bytes that can be read from the Reader
|
||||
// after calling Rewind.
|
||||
Length() int
|
||||
}
|
||||
|
||||
// ByteReader implements a RewindReader for a byte slice.
|
||||
type ByteReader struct {
|
||||
*bytes.Reader
|
||||
Len int
|
||||
}
|
||||
|
||||
// Rewind restarts the reader from the beginning of the data.
|
||||
func (b *ByteReader) Rewind() error {
|
||||
_, err := b.Reader.Seek(0, io.SeekStart)
|
||||
return err
|
||||
}
|
||||
|
||||
// Length returns the number of bytes read from the reader after Rewind is
|
||||
// called.
|
||||
func (b *ByteReader) Length() int {
|
||||
return b.Len
|
||||
}
|
||||
|
||||
// statically ensure that *ByteReader implements RewindReader.
|
||||
var _ RewindReader = &ByteReader{}
|
||||
|
||||
// NewByteReader prepares a ByteReader that can then be used to read buf.
|
||||
func NewByteReader(buf []byte) *ByteReader {
|
||||
return &ByteReader{
|
||||
Reader: bytes.NewReader(buf),
|
||||
Len: len(buf),
|
||||
}
|
||||
}
|
||||
|
||||
// statically ensure that *FileReader implements RewindReader.
|
||||
var _ RewindReader = &FileReader{}
|
||||
|
||||
// FileReader implements a RewindReader for an open file.
|
||||
type FileReader struct {
|
||||
io.ReadSeeker
|
||||
Len int
|
||||
}
|
||||
|
||||
// Rewind seeks to the beginning of the file.
|
||||
func (f *FileReader) Rewind() error {
|
||||
_, err := f.ReadSeeker.Seek(0, io.SeekStart)
|
||||
return errors.Wrap(err, "Seek")
|
||||
}
|
||||
|
||||
// Length returns the length of the file.
|
||||
func (f *FileReader) Length() int {
|
||||
return f.Len
|
||||
}
|
||||
|
||||
// NewFileReader wraps f in a *FileReader.
|
||||
func NewFileReader(f io.ReadSeeker) (*FileReader, error) {
|
||||
pos, err := f.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Seek")
|
||||
}
|
||||
|
||||
fr := &FileReader{
|
||||
ReadSeeker: f,
|
||||
Len: int(pos),
|
||||
}
|
||||
|
||||
err = fr.Rewind()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return fr, nil
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package restic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func TestByteReader(t *testing.T) {
|
||||
buf := []byte("foobar")
|
||||
fn := func() RewindReader {
|
||||
return NewByteReader(buf)
|
||||
}
|
||||
testRewindReader(t, fn, buf)
|
||||
}
|
||||
|
||||
func TestFileReader(t *testing.T) {
|
||||
buf := []byte("foobar")
|
||||
|
||||
d, cleanup := test.TempDir(t)
|
||||
defer cleanup()
|
||||
|
||||
filename := filepath.Join(d, "file-reader-test")
|
||||
err := ioutil.WriteFile(filename, []byte("foobar"), 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f, err := os.Open(filename)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := f.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
fn := func() RewindReader {
|
||||
rd, err := NewFileReader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rd
|
||||
}
|
||||
|
||||
testRewindReader(t, fn, buf)
|
||||
}
|
||||
|
||||
func testRewindReader(t *testing.T, fn func() RewindReader, data []byte) {
|
||||
seed := time.Now().Unix()
|
||||
t.Logf("seed is %d", seed)
|
||||
rnd := rand.New(rand.NewSource(seed))
|
||||
|
||||
type ReaderTestFunc func(t testing.TB, r RewindReader, data []byte)
|
||||
var tests = []ReaderTestFunc{
|
||||
func(t testing.TB, rd RewindReader, data []byte) {
|
||||
if rd.Length() != len(data) {
|
||||
t.Fatalf("wrong length returned, want %d, got %d", len(data), rd.Length())
|
||||
}
|
||||
|
||||
buf := make([]byte, len(data))
|
||||
_, err := io.ReadFull(rd, buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, data) {
|
||||
t.Fatalf("wrong data returned")
|
||||
}
|
||||
|
||||
if rd.Length() != len(data) {
|
||||
t.Fatalf("wrong length returned, want %d, got %d", len(data), rd.Length())
|
||||
}
|
||||
|
||||
err = rd.Rewind()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if rd.Length() != len(data) {
|
||||
t.Fatalf("wrong length returned, want %d, got %d", len(data), rd.Length())
|
||||
}
|
||||
|
||||
buf2 := make([]byte, len(data))
|
||||
_, err = io.ReadFull(rd, buf2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf2, data) {
|
||||
t.Fatalf("wrong data returned")
|
||||
}
|
||||
|
||||
if rd.Length() != len(data) {
|
||||
t.Fatalf("wrong length returned, want %d, got %d", len(data), rd.Length())
|
||||
}
|
||||
},
|
||||
func(t testing.TB, rd RewindReader, data []byte) {
|
||||
// read first bytes
|
||||
buf := make([]byte, rnd.Intn(len(data)))
|
||||
_, err := io.ReadFull(rd, buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf, data[:len(buf)]) {
|
||||
t.Fatalf("wrong data returned")
|
||||
}
|
||||
|
||||
err = rd.Rewind()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
buf2 := make([]byte, rnd.Intn(len(data)))
|
||||
_, err = io.ReadFull(rd, buf2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf2, data[:len(buf2)]) {
|
||||
t.Fatalf("wrong data returned")
|
||||
}
|
||||
|
||||
// read remainder
|
||||
buf3 := make([]byte, len(data)-len(buf2))
|
||||
_, err = io.ReadFull(rd, buf3)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(buf3, data[len(buf2):]) {
|
||||
t.Fatalf("wrong data returned")
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
rd := fn()
|
||||
test(t, rd, data)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user