Move package 'repo' to package 'repository'

repository/blob.go

@@ -0,0 +1,48 @@
+package repository
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/restic/restic/backend"
+)
+
+type Blob struct {
+	ID          backend.ID `json:"id,omitempty"`
+	Size        uint64     `json:"size,omitempty"`
+	Storage     backend.ID `json:"sid,omitempty"`   // encrypted ID
+	StorageSize uint64     `json:"ssize,omitempty"` // encrypted Size
+}
+
+type Blobs []Blob
+
+func (b Blob) Valid() bool {
+	if b.ID == nil || b.Storage == nil || b.StorageSize == 0 {
+		return false
+	}
+
+	return true
+}
+
+func (b Blob) String() string {
+	return fmt.Sprintf("Blob<%s (%d) -> %s (%d)>",
+		b.ID.Str(), b.Size,
+		b.Storage.Str(), b.StorageSize)
+}
+
+// Compare compares two blobs by comparing the ID and the size. It returns -1,
+// 0, or 1.
+func (blob Blob) Compare(other Blob) int {
+	if res := bytes.Compare(other.ID, blob.ID); res != 0 {
+		return res
+	}
+
+	if blob.Size < other.Size {
+		return -1
+	}
+	if blob.Size > other.Size {
+		return 1
+	}
+
+	return 0
+}

repository/doc.go

@@ -0,0 +1,2 @@
+// Package repo implements a restic repository on top of a backend.
+package repository

repository/index.go

@@ -0,0 +1,258 @@
+package repository
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"sync"
+
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/debug"
+	"github.com/restic/restic/pack"
+)
+
+// Index holds a lookup table for id -> pack.
+type Index struct {
+	m    sync.Mutex
+	pack map[string]indexEntry
+}
+
+type indexEntry struct {
+	tpe    pack.BlobType
+	packID backend.ID
+	offset uint
+	length uint
+	old    bool
+}
+
+// NewIndex returns a new index.
+func NewIndex() *Index {
+	return &Index{
+		pack: make(map[string]indexEntry),
+	}
+}
+
+func (idx *Index) store(t pack.BlobType, id, pack backend.ID, offset, length uint, old bool) {
+	idx.pack[id.String()] = indexEntry{
+		tpe:    t,
+		packID: pack,
+		offset: offset,
+		length: length,
+		old:    old,
+	}
+}
+
+// Store remembers the id and pack in the index.
+func (idx *Index) Store(t pack.BlobType, id, pack backend.ID, offset, length uint) {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+
+	debug.Log("Index.Store", "pack %v contains id %v (%v), offset %v, length %v",
+		pack.Str(), id.Str(), t, offset, length)
+
+	idx.store(t, id, pack, offset, length, false)
+}
+
+// Remove removes the pack ID from the index.
+func (idx *Index) Remove(packID backend.ID) {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+
+	debug.Log("Index.Remove", "id %v removed", packID.Str())
+
+	s := packID.String()
+	if _, ok := idx.pack[s]; ok {
+		delete(idx.pack, s)
+	}
+}
+
+// Lookup returns the pack for the id.
+func (idx *Index) Lookup(id backend.ID) (packID backend.ID, tpe pack.BlobType, offset, length uint, err error) {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+
+	if p, ok := idx.pack[id.String()]; ok {
+		debug.Log("Index.Lookup", "id %v found in pack %v at %d, length %d",
+			id.Str(), p.packID.Str(), p.offset, p.length)
+		return p.packID, p.tpe, p.offset, p.length, nil
+	}
+
+	debug.Log("Index.Lookup", "id %v not found", id.Str())
+	return nil, pack.Data, 0, 0, errors.New("id not found")
+}
+
+// Has returns true iff the id is listed in the index.
+func (idx *Index) Has(id backend.ID) bool {
+	_, _, _, _, err := idx.Lookup(id)
+	if err == nil {
+		return true
+	}
+
+	return false
+}
+
+// Merge loads all items from other into idx.
+func (idx *Index) Merge(other *Index) {
+	debug.Log("Index.Merge", "Merge index with %p", other)
+	idx.m.Lock()
+	defer idx.m.Unlock()
+
+	for k, v := range other.pack {
+		if _, ok := idx.pack[k]; ok {
+			debug.Log("Index.Merge", "index already has key %v, updating", k[:8])
+		}
+
+		idx.pack[k] = v
+	}
+	debug.Log("Index.Merge", "done merging index")
+}
+
+// Each returns a channel that yields all blobs known to the index. If done is
+// closed, the background goroutine terminates. This blocks any modification of
+// the index.
+func (idx *Index) Each(done chan struct{}) <-chan pack.Blob {
+	idx.m.Lock()
+
+	ch := make(chan pack.Blob)
+
+	go func() {
+		defer idx.m.Unlock()
+		defer func() {
+			close(ch)
+		}()
+
+		for ids, blob := range idx.pack {
+			id, err := backend.ParseID(ids)
+			if err != nil {
+				// ignore invalid IDs
+				continue
+			}
+
+			select {
+			case <-done:
+				return
+			case ch <- pack.Blob{
+				ID:     id,
+				Offset: blob.offset,
+				Type:   blob.tpe,
+				Length: uint32(blob.length),
+			}:
+			}
+		}
+	}()
+
+	return ch
+}
+
+// Count returns the number of blobs of type t in the index.
+func (idx *Index) Count(t pack.BlobType) (n uint) {
+	debug.Log("Index.Count", "counting blobs of type %v", t)
+	idx.m.Lock()
+	defer idx.m.Unlock()
+
+	for id, blob := range idx.pack {
+		if blob.tpe == t {
+			n++
+			debug.Log("Index.Count", "  blob %v counted: %v", id[:8], blob)
+		}
+	}
+
+	return
+}
+
+type packJSON struct {
+	ID    string     `json:"id"`
+	Blobs []blobJSON `json:"blobs"`
+}
+
+type blobJSON struct {
+	ID     string        `json:"id"`
+	Type   pack.BlobType `json:"type"`
+	Offset uint          `json:"offset"`
+	Length uint          `json:"length"`
+}
+
+// Encode writes the JSON serialization of the index to the writer w. This
+// serialization only contains new blobs added via idx.Store(), not old ones
+// introduced via DecodeIndex().
+func (idx *Index) Encode(w io.Writer) error {
+	debug.Log("Index.Encode", "encoding index")
+	idx.m.Lock()
+	defer idx.m.Unlock()
+
+	list := []*packJSON{}
+	packs := make(map[string]*packJSON)
+
+	for id, blob := range idx.pack {
+		if blob.old {
+			continue
+		}
+
+		debug.Log("Index.Encode", "handle blob %q", id[:8])
+
+		if blob.packID == nil {
+			debug.Log("Index.Encode", "blob %q has no packID! (type %v, offset %v, length %v)",
+				id[:8], blob.tpe, blob.offset, blob.length)
+			return fmt.Errorf("unable to serialize index: pack for blob %v hasn't been written yet", id)
+		}
+
+		// see if pack is already in map
+		p, ok := packs[blob.packID.String()]
+		if !ok {
+			// else create new pack
+			p = &packJSON{ID: blob.packID.String()}
+
+			// and append it to the list and map
+			list = append(list, p)
+			packs[p.ID] = p
+		}
+
+		// add blob
+		p.Blobs = append(p.Blobs, blobJSON{
+			ID:     id,
+			Type:   blob.tpe,
+			Offset: blob.offset,
+			Length: blob.length,
+		})
+	}
+
+	debug.Log("Index.Encode", "done")
+
+	enc := json.NewEncoder(w)
+	return enc.Encode(list)
+}
+
+// DecodeIndex loads and unserializes an index from rd.
+func DecodeIndex(rd io.Reader) (*Index, error) {
+	debug.Log("Index.DecodeIndex", "Start decoding index")
+	list := []*packJSON{}
+
+	dec := json.NewDecoder(rd)
+	err := dec.Decode(&list)
+	if err != nil {
+		return nil, err
+	}
+
+	idx := NewIndex()
+	for _, pack := range list {
+		packID, err := backend.ParseID(pack.ID)
+		if err != nil {
+			debug.Log("Index.DecodeIndex", "error parsing pack ID %q: %v", pack.ID, err)
+			return nil, err
+		}
+
+		for _, blob := range pack.Blobs {
+			blobID, err := backend.ParseID(blob.ID)
+			if err != nil {
+				debug.Log("Index.DecodeIndex", "error parsing blob ID %q: %v", blob.ID, err)
+				return nil, err
+			}
+
+			idx.store(blob.Type, blobID, packID, blob.Offset, blob.Length, true)
+		}
+	}
+
+	debug.Log("Index.DecodeIndex", "done")
+	return idx, err
+}

repository/index_test.go

@@ -0,0 +1,225 @@
+package repository_test
+
+import (
+	"bytes"
+	"crypto/rand"
+	"io"
+	"testing"
+
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/pack"
+	"github.com/restic/restic/repository"
+	. "github.com/restic/restic/test"
+)
+
+func randomID() backend.ID {
+	buf := make([]byte, backend.IDSize)
+	_, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		panic(err)
+	}
+	return buf
+}
+
+func TestIndexSerialize(t *testing.T) {
+	type testEntry struct {
+		id             backend.ID
+		pack           backend.ID
+		tpe            pack.BlobType
+		offset, length uint
+	}
+	tests := []testEntry{}
+
+	idx := repository.NewIndex()
+
+	// create 50 packs with 20 blobs each
+	for i := 0; i < 50; i++ {
+		packID := randomID()
+
+		pos := uint(0)
+		for j := 0; j < 20; j++ {
+			id := randomID()
+			length := uint(i*100 + j)
+			idx.Store(pack.Data, id, packID, pos, length)
+
+			tests = append(tests, testEntry{
+				id:     id,
+				pack:   packID,
+				tpe:    pack.Data,
+				offset: pos,
+				length: length,
+			})
+
+			pos += length
+		}
+	}
+
+	wr := bytes.NewBuffer(nil)
+	err := idx.Encode(wr)
+	OK(t, err)
+
+	idx2, err := repository.DecodeIndex(wr)
+	OK(t, err)
+	Assert(t, idx2 != nil,
+		"nil returned for decoded index")
+
+	wr2 := bytes.NewBuffer(nil)
+	err = idx2.Encode(wr2)
+	OK(t, err)
+
+	for _, testBlob := range tests {
+		packID, tpe, offset, length, err := idx.Lookup(testBlob.id)
+		OK(t, err)
+
+		Equals(t, testBlob.pack, packID)
+		Equals(t, testBlob.tpe, tpe)
+		Equals(t, testBlob.offset, offset)
+		Equals(t, testBlob.length, length)
+
+		packID, tpe, offset, length, err = idx2.Lookup(testBlob.id)
+		OK(t, err)
+
+		Equals(t, testBlob.pack, packID)
+		Equals(t, testBlob.tpe, tpe)
+		Equals(t, testBlob.offset, offset)
+		Equals(t, testBlob.length, length)
+	}
+
+	// add more blobs to idx2
+	newtests := []testEntry{}
+	for i := 0; i < 10; i++ {
+		packID := randomID()
+
+		pos := uint(0)
+		for j := 0; j < 10; j++ {
+			id := randomID()
+			length := uint(i*100 + j)
+			idx2.Store(pack.Data, id, packID, pos, length)
+
+			newtests = append(newtests, testEntry{
+				id:     id,
+				pack:   packID,
+				tpe:    pack.Data,
+				offset: pos,
+				length: length,
+			})
+
+			pos += length
+		}
+	}
+
+	// serialize idx2, unserialize to idx3
+	wr3 := bytes.NewBuffer(nil)
+	err = idx2.Encode(wr3)
+	OK(t, err)
+
+	idx3, err := repository.DecodeIndex(wr3)
+	OK(t, err)
+	Assert(t, idx3 != nil,
+		"nil returned for decoded index")
+
+	// all old blobs must not be present in the index
+	for _, testBlob := range tests {
+		_, _, _, _, err := idx3.Lookup(testBlob.id)
+		Assert(t, err != nil,
+			"found old id %v in serialized index", testBlob.id.Str())
+	}
+
+	// all new blobs must be in the index
+	for _, testBlob := range newtests {
+		packID, tpe, offset, length, err := idx3.Lookup(testBlob.id)
+		OK(t, err)
+
+		Equals(t, testBlob.pack, packID)
+		Equals(t, testBlob.tpe, tpe)
+		Equals(t, testBlob.offset, offset)
+		Equals(t, testBlob.length, length)
+	}
+}
+
+func TestIndexSize(t *testing.T) {
+	idx := repository.NewIndex()
+
+	packs := 200
+	blobs := 100
+	for i := 0; i < packs; i++ {
+		packID := randomID()
+
+		pos := uint(0)
+		for j := 0; j < blobs; j++ {
+			id := randomID()
+			length := uint(i*100 + j)
+			idx.Store(pack.Data, id, packID, pos, length)
+
+			pos += length
+		}
+	}
+
+	wr := bytes.NewBuffer(nil)
+
+	err := idx.Encode(wr)
+	OK(t, err)
+
+	t.Logf("Index file size for %d blobs in %d packs is %d", blobs*packs, packs, wr.Len())
+}
+
+// example index serialization from doc/Design.md
+var docExample = []byte(`
+[ {
+  "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c",
+  "blobs": [
+    {
+      "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce",
+      "type": "data",
+      "offset": 0,
+      "length": 25
+    },{
+      "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae",
+      "type": "tree",
+      "offset": 38,
+      "length": 100
+    },
+    {
+      "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66",
+      "type": "data",
+      "offset": 150,
+      "length": 123
+    }
+  ]
+} ]
+`)
+
+var exampleTests = []struct {
+	id, packID     backend.ID
+	tpe            pack.BlobType
+	offset, length uint
+}{
+	{
+		ParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"),
+		ParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"),
+		pack.Data, 0, 25,
+	}, {
+		ParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"),
+		ParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"),
+		pack.Tree, 38, 100,
+	}, {
+		ParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"),
+		ParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"),
+		pack.Data, 150, 123,
+	},
+}
+
+func TestIndexUnserialize(t *testing.T) {
+	idx, err := repository.DecodeIndex(bytes.NewReader(docExample))
+	OK(t, err)
+
+	for _, test := range exampleTests {
+		packID, tpe, offset, length, err := idx.Lookup(test.id)
+		OK(t, err)
+
+		Equals(t, test.packID, packID)
+		Equals(t, test.tpe, tpe)
+		Equals(t, test.offset, offset)
+		Equals(t, test.length, length)
+	}
+}

repository/key.go

@@ -0,0 +1,228 @@
+package repository
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"os/user"
+	"time"
+
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/crypto"
+)
+
+var (
+	// ErrNoKeyFound is returned when no key for the repository could be decrypted.
+	ErrNoKeyFound = errors.New("no key could be found")
+)
+
+// TODO: figure out scrypt values on the fly depending on the current
+// hardware.
+const (
+	scryptN        = 65536
+	scryptR        = 8
+	scryptP        = 1
+	scryptSaltsize = 64
+)
+
+// Key represents an encrypted master key for a repository.
+type Key struct {
+	Created  time.Time `json:"created"`
+	Username string    `json:"username"`
+	Hostname string    `json:"hostname"`
+
+	KDF  string `json:"kdf"`
+	N    int    `json:"N"`
+	R    int    `json:"r"`
+	P    int    `json:"p"`
+	Salt []byte `json:"salt"`
+	Data []byte `json:"data"`
+
+	user   *crypto.Key
+	master *crypto.Key
+
+	name string
+}
+
+// createMasterKey creates a new master key in the given backend and encrypts
+// it with the password.
+func createMasterKey(s *Repo, password string) (*Key, error) {
+	return AddKey(s, password, nil)
+}
+
+// OpenKey tries do decrypt the key specified by name with the given password.
+func OpenKey(s *Repo, name string, password string) (*Key, error) {
+	k, err := LoadKey(s, name)
+	if err != nil {
+		return nil, err
+	}
+
+	// check KDF
+	if k.KDF != "scrypt" {
+		return nil, errors.New("only supported KDF is scrypt()")
+	}
+
+	// derive user key
+	k.user, err = crypto.KDF(k.N, k.R, k.P, k.Salt, password)
+	if err != nil {
+		return nil, err
+	}
+
+	// decrypt master keys
+	buf, err := crypto.Decrypt(k.user, []byte{}, k.Data)
+	if err != nil {
+		return nil, err
+	}
+
+	// restore json
+	k.master = &crypto.Key{}
+	err = json.Unmarshal(buf, k.master)
+	if err != nil {
+		return nil, err
+	}
+	k.name = name
+
+	if !k.Valid() {
+		return nil, errors.New("Invalid key for repository")
+	}
+
+	return k, nil
+}
+
+// SearchKey tries to decrypt all keys in the backend with the given password.
+// If none could be found, ErrNoKeyFound is returned.
+func SearchKey(s *Repo, password string) (*Key, error) {
+	// try all keys in repo
+	done := make(chan struct{})
+	defer close(done)
+	for name := range s.List(backend.Key, done) {
+		key, err := OpenKey(s, name, password)
+		if err != nil {
+			continue
+		}
+
+		return key, nil
+	}
+
+	return nil, ErrNoKeyFound
+}
+
+// LoadKey loads a key from the backend.
+func LoadKey(s *Repo, name string) (*Key, error) {
+	// extract data from repo
+	rd, err := s.be.Get(backend.Key, name)
+	if err != nil {
+		return nil, err
+	}
+	defer rd.Close()
+
+	// restore json
+	dec := json.NewDecoder(rd)
+	k := Key{}
+	err = dec.Decode(&k)
+	if err != nil {
+		return nil, err
+	}
+
+	return &k, nil
+}
+
+// AddKey adds a new key to an already existing repository.
+func AddKey(s *Repo, password string, template *crypto.Key) (*Key, error) {
+	// fill meta data about key
+	newkey := &Key{
+		Created: time.Now(),
+		KDF:     "scrypt",
+		N:       scryptN,
+		R:       scryptR,
+		P:       scryptP,
+	}
+
+	hn, err := os.Hostname()
+	if err == nil {
+		newkey.Hostname = hn
+	}
+
+	usr, err := user.Current()
+	if err == nil {
+		newkey.Username = usr.Username
+	}
+
+	// generate random salt
+	newkey.Salt = make([]byte, scryptSaltsize)
+	n, err := rand.Read(newkey.Salt)
+	if n != scryptSaltsize || err != nil {
+		panic("unable to read enough random bytes for salt")
+	}
+
+	// call KDF to derive user key
+	newkey.user, err = crypto.KDF(newkey.N, newkey.R, newkey.P, newkey.Salt, password)
+	if err != nil {
+		return nil, err
+	}
+
+	if template == nil {
+		// generate new random master keys
+		newkey.master = crypto.NewRandomKey()
+	} else {
+		// copy master keys from old key
+		newkey.master = template
+	}
+
+	// encrypt master keys (as json) with user key
+	buf, err := json.Marshal(newkey.master)
+	if err != nil {
+		return nil, err
+	}
+
+	newkey.Data, err = crypto.Encrypt(newkey.user, nil, buf)
+
+	// dump as json
+	buf, err = json.Marshal(newkey)
+	if err != nil {
+		return nil, err
+	}
+
+	// store in repository and return
+	blob, err := s.be.Create()
+	if err != nil {
+		return nil, err
+	}
+
+	plainhw := backend.NewHashingWriter(blob, sha256.New())
+
+	_, err = plainhw.Write(buf)
+	if err != nil {
+		return nil, err
+	}
+
+	name := backend.ID(plainhw.Sum(nil)).String()
+
+	err = blob.Finalize(backend.Key, name)
+	if err != nil {
+		return nil, err
+	}
+
+	newkey.name = name
+
+	return newkey, nil
+}
+
+func (k *Key) String() string {
+	if k == nil {
+		return "<Key nil>"
+	}
+	return fmt.Sprintf("<Key of %s@%s, created on %s>", k.Username, k.Hostname, k.Created)
+}
+
+func (k Key) Name() string {
+	return k.name
+}
+
+// Valid tests whether the mac and encryption keys are valid (i.e. not zero)
+func (k *Key) Valid() bool {
+	return k.user.Valid() && k.master.Valid()
+}

repository/pool.go

@@ -0,0 +1,21 @@
+package repository
+
+import (
+	"sync"
+
+	"github.com/restic/restic/chunker"
+)
+
+var bufPool = sync.Pool{
+	New: func() interface{} {
+		return make([]byte, chunker.MinSize)
+	},
+}
+
+func getBuf() []byte {
+	return bufPool.Get().([]byte)
+}
+
+func freeBuf(data []byte) {
+	bufPool.Put(data)
+}

repository/repository.go

@@ -0,0 +1,672 @@
+package repository
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"sync"
+
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/chunker"
+	"github.com/restic/restic/crypto"
+	"github.com/restic/restic/debug"
+	"github.com/restic/restic/pack"
+)
+
+// Config contains the configuration for a repository.
+type Config struct {
+	Version           uint        `json:"version"`
+	ID                string      `json:"id"`
+	ChunkerPolynomial chunker.Pol `json:"chunker_polynomial"`
+}
+
+// Repo is used to access a repository in a backend.
+type Repo struct {
+	be      backend.Backend
+	Config  Config
+	key     *crypto.Key
+	keyName string
+	idx     *Index
+
+	pm    sync.Mutex
+	packs []*pack.Packer
+}
+
+func New(be backend.Backend) *Repo {
+	return &Repo{
+		be:  be,
+		idx: NewIndex(),
+	}
+}
+
+// Find loads the list of all blobs of type t and searches for names which start
+// with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. If
+// more than one is found, nil and ErrMultipleIDMatches is returned.
+func (s *Repo) Find(t backend.Type, prefix string) (string, error) {
+	return backend.Find(s.be, t, prefix)
+}
+
+// FindSnapshot takes a string and tries to find a snapshot whose ID matches
+// the string as closely as possible.
+func (s *Repo) FindSnapshot(name string) (string, error) {
+	return backend.FindSnapshot(s.be, name)
+}
+
+// PrefixLength returns the number of bytes required so that all prefixes of
+// all IDs of type t are unique.
+func (s *Repo) PrefixLength(t backend.Type) (int, error) {
+	return backend.PrefixLength(s.be, t)
+}
+
+// Load tries to load and decrypt content identified by t and id from the
+// backend.
+func (s *Repo) Load(t backend.Type, id backend.ID) ([]byte, error) {
+	debug.Log("Repo.Load", "load %v with id %v", t, id.Str())
+
+	// load blob from pack
+	rd, err := s.be.Get(t, id.String())
+	if err != nil {
+		debug.Log("Repo.Load", "error loading %v: %v", id.Str(), err)
+		return nil, err
+	}
+
+	buf, err := ioutil.ReadAll(rd)
+	if err != nil {
+		return nil, err
+	}
+
+	err = rd.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// check hash
+	if !backend.Hash(buf).Equal(id) {
+		return nil, errors.New("invalid data returned")
+	}
+
+	// decrypt
+	plain, err := s.Decrypt(buf)
+	if err != nil {
+		return nil, err
+	}
+
+	return plain, nil
+}
+
+// LoadBlob tries to load and decrypt content identified by t and id from a
+// pack from the backend.
+func (s *Repo) LoadBlob(t pack.BlobType, id backend.ID) ([]byte, error) {
+	debug.Log("Repo.LoadBlob", "load %v with id %v", t, id.Str())
+	// lookup pack
+	packID, tpe, offset, length, err := s.idx.Lookup(id)
+	if err != nil {
+		debug.Log("Repo.LoadBlob", "id %v not found in index: %v", id.Str(), err)
+		return nil, err
+	}
+
+	if tpe != t {
+		debug.Log("Repo.LoadBlob", "wrong type returned for %v: wanted %v, got %v", id.Str(), t, tpe)
+		return nil, fmt.Errorf("blob has wrong type %v (wanted: %v)", tpe, t)
+	}
+
+	debug.Log("Repo.LoadBlob", "id %v found in pack %v at offset %v (length %d)", id.Str(), packID.Str(), offset, length)
+
+	// load blob from pack
+	rd, err := s.be.GetReader(backend.Data, packID.String(), offset, length)
+	if err != nil {
+		debug.Log("Repo.LoadBlob", "error loading pack %v for %v: %v", packID.Str(), id.Str(), err)
+		return nil, err
+	}
+
+	buf, err := ioutil.ReadAll(rd)
+	if err != nil {
+		return nil, err
+	}
+
+	err = rd.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// decrypt
+	plain, err := s.Decrypt(buf)
+	if err != nil {
+		return nil, err
+	}
+
+	// check hash
+	if !backend.Hash(plain).Equal(id) {
+		return nil, errors.New("invalid data returned")
+	}
+
+	return plain, nil
+}
+
+// LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on
+// the item.
+func (s *Repo) LoadJSONUnpacked(t backend.Type, id backend.ID, item interface{}) error {
+	// load blob from backend
+	rd, err := s.be.Get(t, id.String())
+	if err != nil {
+		return err
+	}
+	defer rd.Close()
+
+	// decrypt
+	decryptRd, err := crypto.DecryptFrom(s.key, rd)
+	defer decryptRd.Close()
+	if err != nil {
+		return err
+	}
+
+	// decode
+	decoder := json.NewDecoder(decryptRd)
+	err = decoder.Decode(item)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// LoadJSONPack calls LoadBlob() to load a blob from the backend, decrypt the
+// data and afterwards call json.Unmarshal on the item.
+func (s *Repo) LoadJSONPack(t pack.BlobType, id backend.ID, item interface{}) error {
+	// lookup pack
+	packID, _, offset, length, err := s.idx.Lookup(id)
+	if err != nil {
+		return err
+	}
+
+	// load blob from pack
+	rd, err := s.be.GetReader(backend.Data, packID.String(), offset, length)
+	if err != nil {
+		return err
+	}
+	defer rd.Close()
+
+	// decrypt
+	decryptRd, err := crypto.DecryptFrom(s.key, rd)
+	defer decryptRd.Close()
+	if err != nil {
+		return err
+	}
+
+	// decode
+	decoder := json.NewDecoder(decryptRd)
+	err = decoder.Decode(item)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+const minPackSize = 4 * chunker.MiB
+const maxPackSize = 16 * chunker.MiB
+const maxPackers = 200
+
+// findPacker returns a packer for a new blob of size bytes. Either a new one is
+// created or one is returned that already has some blobs.
+func (s *Repo) findPacker(size uint) (*pack.Packer, error) {
+	s.pm.Lock()
+	defer s.pm.Unlock()
+
+	// search for a suitable packer
+	if len(s.packs) > 0 {
+		debug.Log("Repo.findPacker", "searching packer for %d bytes\n", size)
+		for i, p := range s.packs {
+			if p.Size()+size < maxPackSize {
+				debug.Log("Repo.findPacker", "found packer %v", p)
+				// remove from list
+				s.packs = append(s.packs[:i], s.packs[i+1:]...)
+				return p, nil
+			}
+		}
+	}
+
+	// no suitable packer found, return new
+	blob, err := s.be.Create()
+	if err != nil {
+		return nil, err
+	}
+	debug.Log("Repo.findPacker", "create new pack %p", blob)
+	return pack.NewPacker(s.key, blob), nil
+}
+
+// insertPacker appends p to s.packs.
+func (s *Repo) insertPacker(p *pack.Packer) {
+	s.pm.Lock()
+	defer s.pm.Unlock()
+
+	s.packs = append(s.packs, p)
+	debug.Log("Repo.insertPacker", "%d packers\n", len(s.packs))
+}
+
+// savePacker stores p in the backend.
+func (s *Repo) savePacker(p *pack.Packer) error {
+	debug.Log("Repo.savePacker", "save packer with %d blobs\n", p.Count())
+	_, err := p.Finalize()
+	if err != nil {
+		return err
+	}
+
+	// move file to the final location
+	sid := p.ID()
+	err = p.Writer().(backend.Blob).Finalize(backend.Data, sid.String())
+	if err != nil {
+		debug.Log("Repo.savePacker", "blob Finalize() error: %v", err)
+		return err
+	}
+
+	debug.Log("Repo.savePacker", "saved as %v", sid.Str())
+
+	// update blobs in the index
+	for _, b := range p.Blobs() {
+		debug.Log("Repo.savePacker", "  updating blob %v to pack %v", b.ID.Str(), sid.Str())
+		s.idx.Store(b.Type, b.ID, sid, b.Offset, uint(b.Length))
+	}
+
+	return nil
+}
+
+// countPacker returns the number of open (unfinished) packers.
+func (s *Repo) countPacker() int {
+	s.pm.Lock()
+	defer s.pm.Unlock()
+
+	return len(s.packs)
+}
+
+// Save encrypts data and stores it to the backend as type t. If data is small
+// enough, it will be packed together with other small blobs.
+func (s *Repo) Save(t pack.BlobType, data []byte, id backend.ID) (backend.ID, error) {
+	if id == nil {
+		// compute plaintext hash
+		id = backend.Hash(data)
+	}
+
+	debug.Log("Repo.Save", "save id %v (%v, %d bytes)", id.Str(), t, len(data))
+
+	// get buf from the pool
+	ciphertext := getBuf()
+	defer freeBuf(ciphertext)
+
+	// encrypt blob
+	ciphertext, err := s.Encrypt(ciphertext, data)
+	if err != nil {
+		return nil, err
+	}
+
+	// find suitable packer and add blob
+	packer, err := s.findPacker(uint(len(ciphertext)))
+	if err != nil {
+		return nil, err
+	}
+
+	// save ciphertext
+	packer.Add(t, id, bytes.NewReader(ciphertext))
+
+	// add this id to the index, although we don't know yet in which pack it
+	// will be saved, the entry will be updated when the pack is written.
+	s.idx.Store(t, id, nil, 0, 0)
+	debug.Log("Repo.Save", "saving stub for %v (%v) in index", id.Str, t)
+
+	// if the pack is not full enough and there are less than maxPackers
+	// packers, put back to the list
+	if packer.Size() < minPackSize && s.countPacker() < maxPackers {
+		debug.Log("Repo.Save", "pack is not full enough (%d bytes)", packer.Size())
+		s.insertPacker(packer)
+		return id, nil
+	}
+
+	// else write the pack to the backend
+	return id, s.savePacker(packer)
+}
+
+// SaveFrom encrypts data read from rd and stores it in a pack in the backend as type t.
+func (s *Repo) SaveFrom(t pack.BlobType, id backend.ID, length uint, rd io.Reader) error {
+	debug.Log("Repo.SaveFrom", "save id %v (%v, %d bytes)", id.Str(), t, length)
+	if id == nil {
+		return errors.New("id is nil")
+	}
+
+	buf, err := ioutil.ReadAll(rd)
+	if err != nil {
+		return err
+	}
+
+	_, err = s.Save(t, buf, id)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SaveJSON serialises item as JSON and encrypts and saves it in a pack in the
+// backend as type t.
+func (s *Repo) SaveJSON(t pack.BlobType, item interface{}) (backend.ID, error) {
+	debug.Log("Repo.SaveJSON", "save %v blob", t)
+	buf := getBuf()[:0]
+	defer freeBuf(buf)
+
+	wr := bytes.NewBuffer(buf)
+
+	enc := json.NewEncoder(wr)
+	err := enc.Encode(item)
+	if err != nil {
+		return nil, fmt.Errorf("json.Encode: %v", err)
+	}
+
+	buf = wr.Bytes()
+	return s.Save(t, buf, nil)
+}
+
+// SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the
+// backend as type t, without a pack. It returns the storage hash.
+func (s *Repo) SaveJSONUnpacked(t backend.Type, item interface{}) (backend.ID, error) {
+	// create file
+	blob, err := s.be.Create()
+	if err != nil {
+		return nil, err
+	}
+	debug.Log("Repo.SaveJSONUnpacked", "create new file %p", blob)
+
+	// hash
+	hw := backend.NewHashingWriter(blob, sha256.New())
+
+	// encrypt blob
+	ewr := crypto.EncryptTo(s.key, hw)
+
+	enc := json.NewEncoder(ewr)
+	err = enc.Encode(item)
+	if err != nil {
+		return nil, fmt.Errorf("json.Encode: %v", err)
+	}
+
+	err = ewr.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// finalize blob in the backend
+	sid := backend.ID(hw.Sum(nil))
+
+	err = blob.Finalize(t, sid.String())
+	if err != nil {
+		return nil, err
+	}
+
+	return sid, nil
+}
+
+// Flush saves all remaining packs.
+func (s *Repo) Flush() error {
+	s.pm.Lock()
+	defer s.pm.Unlock()
+
+	debug.Log("Repo.Flush", "manually flushing %d packs", len(s.packs))
+
+	for _, p := range s.packs {
+		err := s.savePacker(p)
+		if err != nil {
+			return err
+		}
+	}
+	s.packs = s.packs[:0]
+
+	return nil
+}
+
+func (s *Repo) Backend() backend.Backend {
+	return s.be
+}
+
+func (s *Repo) Index() *Index {
+	return s.idx
+}
+
+// SetIndex instructs the repository to use the given index.
+func (s *Repo) SetIndex(i *Index) {
+	s.idx = i
+}
+
+// SaveIndex saves all new packs in the index in the backend, returned is the
+// storage ID.
+func (s *Repo) SaveIndex() (backend.ID, error) {
+	debug.Log("Repo.SaveIndex", "Saving index")
+
+	// create blob
+	blob, err := s.be.Create()
+	if err != nil {
+		return nil, err
+	}
+
+	debug.Log("Repo.SaveIndex", "create new pack %p", blob)
+
+	// hash
+	hw := backend.NewHashingWriter(blob, sha256.New())
+
+	// encrypt blob
+	ewr := crypto.EncryptTo(s.key, hw)
+
+	err = s.idx.Encode(ewr)
+	if err != nil {
+		return nil, err
+	}
+
+	err = ewr.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// finalize blob in the backend
+	sid := backend.ID(hw.Sum(nil))
+
+	err = blob.Finalize(backend.Index, sid.String())
+	if err != nil {
+		return nil, err
+	}
+
+	debug.Log("Repo.SaveIndex", "Saved index as %v", sid.Str())
+
+	return sid, nil
+}
+
+// LoadIndex loads all index files from the backend and merges them with the
+// current index.
+func (s *Repo) LoadIndex() error {
+	debug.Log("Repo.LoadIndex", "Loading index")
+	done := make(chan struct{})
+	defer close(done)
+
+	for id := range s.be.List(backend.Index, done) {
+		err := s.loadIndex(id)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// loadIndex loads the index id and merges it with the currently used index.
+func (s *Repo) loadIndex(id string) error {
+	debug.Log("Repo.loadIndex", "Loading index %v", id[:8])
+	before := len(s.idx.pack)
+
+	rd, err := s.be.Get(backend.Index, id)
+	defer rd.Close()
+	if err != nil {
+		return err
+	}
+
+	// decrypt
+	decryptRd, err := crypto.DecryptFrom(s.key, rd)
+	defer decryptRd.Close()
+	if err != nil {
+		return err
+	}
+
+	idx, err := DecodeIndex(decryptRd)
+	if err != nil {
+		debug.Log("Repo.loadIndex", "error while decoding index %v: %v", id, err)
+		return err
+	}
+
+	s.idx.Merge(idx)
+
+	after := len(s.idx.pack)
+	debug.Log("Repo.loadIndex", "Loaded index %v, added %v blobs", id[:8], after-before)
+
+	return nil
+}
+
+const repositoryIDSize = sha256.Size
+const RepoVersion = 1
+
+func createConfig(s *Repo) (err error) {
+	s.Config.ChunkerPolynomial, err = chunker.RandomPolynomial()
+	if err != nil {
+		return err
+	}
+
+	newID := make([]byte, repositoryIDSize)
+	_, err = io.ReadFull(rand.Reader, newID)
+	if err != nil {
+		return err
+	}
+
+	s.Config.ID = hex.EncodeToString(newID)
+	s.Config.Version = RepoVersion
+
+	debug.Log("Repo.createConfig", "New config: %#v", s.Config)
+
+	_, err = s.SaveJSONUnpacked(backend.Config, s.Config)
+	return err
+}
+
+func (s *Repo) loadConfig(cfg *Config) error {
+	err := s.LoadJSONUnpacked(backend.Config, nil, cfg)
+	if err != nil {
+		return err
+	}
+
+	if cfg.Version != RepoVersion {
+		return errors.New("unsupported repository version")
+	}
+
+	if !cfg.ChunkerPolynomial.Irreducible() {
+		return errors.New("invalid chunker polynomial")
+	}
+
+	return nil
+}
+
+// SearchKey finds a key with the supplied password, afterwards the config is
+// read and parsed.
+func (s *Repo) SearchKey(password string) error {
+	key, err := SearchKey(s, password)
+	if err != nil {
+		return err
+	}
+
+	s.key = key.master
+	s.keyName = key.Name()
+	return s.loadConfig(&s.Config)
+}
+
+// Init creates a new master key with the supplied password and initializes the
+// repository config.
+func (s *Repo) Init(password string) error {
+	has, err := s.Test(backend.Config, "")
+	if err != nil {
+		return err
+	}
+	if has {
+		return errors.New("repository master key and config already initialized")
+	}
+
+	key, err := createMasterKey(s, password)
+	if err != nil {
+		return err
+	}
+
+	s.key = key.master
+	s.keyName = key.Name()
+	return createConfig(s)
+}
+
+func (s *Repo) Decrypt(ciphertext []byte) ([]byte, error) {
+	if s.key == nil {
+		return nil, errors.New("key for repository not set")
+	}
+
+	return crypto.Decrypt(s.key, nil, ciphertext)
+}
+
+func (s *Repo) Encrypt(ciphertext, plaintext []byte) ([]byte, error) {
+	if s.key == nil {
+		return nil, errors.New("key for repository not set")
+	}
+
+	return crypto.Encrypt(s.key, ciphertext, plaintext)
+}
+
+func (s *Repo) Key() *crypto.Key {
+	return s.key
+}
+
+func (s *Repo) KeyName() string {
+	return s.keyName
+}
+
+// Count returns the number of blobs of a given type in the backend.
+func (s *Repo) Count(t backend.Type) (n uint) {
+	for _ = range s.be.List(t, nil) {
+		n++
+	}
+
+	return
+}
+
+// Proxy methods to backend
+
+func (s *Repo) Get(t backend.Type, name string) (io.ReadCloser, error) {
+	return s.be.Get(t, name)
+}
+
+func (s *Repo) List(t backend.Type, done <-chan struct{}) <-chan string {
+	return s.be.List(t, done)
+}
+
+func (s *Repo) Test(t backend.Type, name string) (bool, error) {
+	return s.be.Test(t, name)
+}
+
+func (s *Repo) Remove(t backend.Type, name string) error {
+	return s.be.Remove(t, name)
+}
+
+func (s *Repo) Close() error {
+	return s.be.Close()
+}
+
+func (s *Repo) Delete() error {
+	if b, ok := s.be.(backend.Deleter); ok {
+		return b.Delete()
+	}
+
+	return errors.New("Delete() called for backend that does not implement this method")
+}
+
+func (s *Repo) Location() string {
+	return s.be.Location()
+}

repository/repository_test.go

@@ -0,0 +1,199 @@
+package repository_test
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/json"
+	"flag"
+	"io"
+	"testing"
+
+	"github.com/restic/restic"
+	"github.com/restic/restic/backend"
+	"github.com/restic/restic/pack"
+	. "github.com/restic/restic/test"
+)
+
+var benchTestDir = flag.String("test.dir", ".", "dir used in benchmarks (default: .)")
+
+type testJSONStruct struct {
+	Foo uint32
+	Bar string
+	Baz []byte
+}
+
+var repoTests = []testJSONStruct{
+	testJSONStruct{Foo: 23, Bar: "Teststring", Baz: []byte("xx")},
+}
+
+func TestSaveJSON(t *testing.T) {
+	repo := SetupRepo(t)
+	defer TeardownRepo(t, repo)
+
+	for _, obj := range repoTests {
+		data, err := json.Marshal(obj)
+		OK(t, err)
+		data = append(data, '\n')
+		h := sha256.Sum256(data)
+
+		id, err := repo.SaveJSON(pack.Tree, obj)
+		OK(t, err)
+
+		Assert(t, bytes.Equal(h[:], id),
+			"TestSaveJSON: wrong plaintext ID: expected %02x, got %02x",
+			h, id)
+	}
+}
+
+func BenchmarkSaveJSON(t *testing.B) {
+	repo := SetupRepo(t)
+	defer TeardownRepo(t, repo)
+
+	obj := repoTests[0]
+
+	data, err := json.Marshal(obj)
+	OK(t, err)
+	data = append(data, '\n')
+	h := sha256.Sum256(data)
+
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		id, err := repo.SaveJSON(pack.Tree, obj)
+		OK(t, err)
+
+		Assert(t, bytes.Equal(h[:], id),
+			"TestSaveJSON: wrong plaintext ID: expected %02x, got %02x",
+			h, id)
+	}
+}
+
+var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20}
+
+func TestSave(t *testing.T) {
+	repo := SetupRepo(t)
+	defer TeardownRepo(t, repo)
+
+	for _, size := range testSizes {
+		data := make([]byte, size)
+		_, err := io.ReadFull(rand.Reader, data)
+		OK(t, err)
+
+		id := backend.Hash(data)
+
+		// save
+		sid, err := repo.Save(pack.Data, data, nil)
+		OK(t, err)
+
+		Equals(t, id, sid)
+
+		OK(t, repo.Flush())
+
+		// read back
+		buf, err := repo.LoadBlob(pack.Data, id)
+
+		Assert(t, len(buf) == len(data),
+			"number of bytes read back does not match: expected %d, got %d",
+			len(data), len(buf))
+
+		Assert(t, bytes.Equal(buf, data),
+			"data does not match: expected %02x, got %02x",
+			data, buf)
+	}
+}
+
+func TestSaveFrom(t *testing.T) {
+	repo := SetupRepo(t)
+	defer TeardownRepo(t, repo)
+
+	for _, size := range testSizes {
+		data := make([]byte, size)
+		_, err := io.ReadFull(rand.Reader, data)
+		OK(t, err)
+
+		id := backend.Hash(data)
+
+		// save
+		err = repo.SaveFrom(pack.Data, id[:], uint(size), bytes.NewReader(data))
+		OK(t, err)
+
+		OK(t, repo.Flush())
+
+		// read back
+		buf, err := repo.LoadBlob(pack.Data, id[:])
+
+		Assert(t, len(buf) == len(data),
+			"number of bytes read back does not match: expected %d, got %d",
+			len(data), len(buf))
+
+		Assert(t, bytes.Equal(buf, data),
+			"data does not match: expected %02x, got %02x",
+			data, buf)
+	}
+}
+
+func BenchmarkSaveFrom(t *testing.B) {
+	repo := SetupRepo(t)
+	defer TeardownRepo(t, repo)
+
+	size := 4 << 20 // 4MiB
+
+	data := make([]byte, size)
+	_, err := io.ReadFull(rand.Reader, data)
+	OK(t, err)
+
+	id := sha256.Sum256(data)
+
+	t.ResetTimer()
+	t.SetBytes(int64(size))
+
+	for i := 0; i < t.N; i++ {
+		// save
+		err = repo.SaveFrom(pack.Data, id[:], uint(size), bytes.NewReader(data))
+		OK(t, err)
+	}
+}
+
+func TestLoadJSONPack(t *testing.T) {
+	if *benchTestDir == "" {
+		t.Skip("benchdir not set, skipping")
+	}
+
+	repo := SetupRepo(t)
+	defer TeardownRepo(t, repo)
+
+	// archive a few files
+	sn := SnapshotDir(t, repo, *benchTestDir, nil)
+	OK(t, repo.Flush())
+
+	tree := restic.NewTree()
+	err := repo.LoadJSONPack(pack.Tree, sn.Tree, &tree)
+	OK(t, err)
+}
+
+func TestLoadJSONUnpacked(t *testing.T) {
+	if *benchTestDir == "" {
+		t.Skip("benchdir not set, skipping")
+	}
+
+	repo := SetupRepo(t)
+	defer TeardownRepo(t, repo)
+
+	// archive a snapshot
+	sn := restic.Snapshot{}
+	sn.Hostname = "foobar"
+	sn.Username = "test!"
+
+	id, err := repo.SaveJSONUnpacked(backend.Snapshot, &sn)
+	OK(t, err)
+
+	var sn2 restic.Snapshot
+
+	// restore
+	err = repo.LoadJSONUnpacked(backend.Snapshot, id, &sn2)
+	OK(t, err)
+
+	Equals(t, sn.Hostname, sn2.Hostname)
+	Equals(t, sn.Username, sn2.Username)
+}