feature (pkg/files): support for delta download

server/pkg/files/handler_cat.go

@@ -1,12 +1,12 @@
 package files
 
 import (
+	"crypto/sha256"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -19,7 +19,9 @@ var file_cache AppCache
 func init() {
 	file_cache = NewAppCache()
 	file_cache.OnEvict(func(key string, value interface{}) {
-		os.RemoveAll(filepath.Join(GetAbsolutePath(TMP_PATH), key))
+		if tmpPath, _, ok := strings.Cut(value.(string), "::"); ok {
+			os.RemoveAll(tmpPath)
+		}
 	})
 }
 
@@ -63,11 +65,16 @@ func FileCat(ctx *App, res http.ResponseWriter, req *http.Request) {
 	}
 
 	// use our cache if necessary (range request) when possible
+	var mtime string
 	if req.Header.Get("range") != "" {
+		if finfo, err := ctx.Backend.Stat(path); err == nil {
+			mtime = fmt.Sprintf("%d", finfo.ModTime().Unix())
+		}
 		ctx.Session["fullpath"] = path
 		if p := file_cache.Get(ctx.Session); p != nil {
-			f, err := os.OpenFile(p.(string), os.O_RDONLY, os.ModePerm)
-			if err == nil {
+			if tmpPath, cachedMtime, _ := strings.Cut(p.(string), "::"); cachedMtime != mtime {
+				file_cache.Del(ctx.Session)
+			} else if f, err := os.OpenFile(tmpPath, os.O_RDONLY, os.ModePerm); err == nil {
 				file = f
 				if fi, err := f.Stat(); err == nil {
 					contentLength = fi.Size()
@@ -161,7 +168,7 @@ func FileCat(ctx *App, res http.ResponseWriter, req *http.Request) {
 				SendErrorResult(res, err)
 				return
 			}
-			file_cache.Set(ctx.Session, tmpPath)
+			file_cache.Set(ctx.Session, tmpPath+"::"+mtime)
 			if _, err = io.Copy(f, file); err != nil {
 				f.Close()
 				file.Close()
@@ -243,32 +250,43 @@ func FileCat(ctx *App, res http.ResponseWriter, req *http.Request) {
 		header.Set("Content-Disposition", "attachment; filename=\""+fname+"\"")
 	}
 	header.Set("Accept-Ranges", "bytes")
+	header.Set("Vary", "Accept")
 
-	if req.Method != http.MethodHead {
-		size := 32
-		if thumb != "true" {
-			switch Config.Get("general.buffer_size").String() {
-			case "small":
-				size = 32
-			case "medium":
-				size = 128
-			case "large":
-				size = 2 * 1024
-			}
+	if req.Method == http.MethodHead {
+		file.Close()
+		return
+	}
+	size := 32
+	if thumb != "true" {
+		switch Config.Get("general.buffer_size").String() {
+		case "small":
+			size = 32
+		case "medium":
+			size = 128
+		case "large":
+			size = 2 * 1024
 		}
-		buf := make([]byte, size*1024)
-		if f, ok := file.(io.ReadSeeker); ok && len(ranges) > 0 {
-			if _, err = f.Seek(ranges[0][0], io.SeekStart); err == nil {
-				header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", ranges[0][0], ranges[0][1], contentLength))
-				header.Set("Content-Length", fmt.Sprintf("%d", ranges[0][1]-ranges[0][0]+1))
-				res.WriteHeader(http.StatusPartialContent)
-				io.CopyBuffer(res, io.LimitReader(f, ranges[0][1]-ranges[0][0]+1), buf)
-			} else {
-				res.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
-			}
+	}
+	buf := make([]byte, size*1024)
+	if f, ok := file.(io.ReadSeeker); ok && len(ranges) > 0 {
+		if _, err = f.Seek(ranges[0][0], io.SeekStart); err == nil {
+			header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", ranges[0][0], ranges[0][1], contentLength))
+			header.Set("Content-Length", fmt.Sprintf("%d", ranges[0][1]-ranges[0][0]+1))
+			res.WriteHeader(http.StatusPartialContent)
+			io.CopyBuffer(res, io.LimitReader(f, ranges[0][1]-ranges[0][0]+1), buf)
 		} else {
-			io.CopyBuffer(res, file, buf)
+			res.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
 		}
+	} else if strings.Contains(req.Header.Get("Accept"), "application/vnd.filestash.delta.rdiff") {
+		header.Set("Cache-Control", "no-store")
+		header.Set("Content-Type", "application/vnd.filestash.delta.rdiff")
+		header.Del("Content-Length")
+		hasher := sha256.New()
+		if err = rdiffSignature(io.TeeReader(file, hasher), res); err == nil {
+			_, err = res.Write(hasher.Sum(nil))
+		}
+	} else {
+		io.CopyBuffer(res, file, buf)
 	}
 	file.Close()
 }

server/pkg/files/handler_save.go

@@ -6,12 +6,10 @@ import (
 	"crypto/sha1"
 	"crypto/sha256"
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"hash"
 	"hash/crc32"
 	"io"
-	"math"
 	"net/http"
 	"path/filepath"
 	"strconv"
@@ -20,8 +18,6 @@ import (
 
 	. "github.com/mickael-kerjean/filestash/server/common"
 	"github.com/mickael-kerjean/filestash/server/pkg/permissions"
-
-	"github.com/balena-os/librsync-go"
 )
 
 var chunkedUploadCache AppCache
@@ -285,18 +281,6 @@ func handlerRDIFF(ctx *App, res http.ResponseWriter, req *http.Request, path str
 		return
 	}
 	defer remote.Close()
-	base := &rdiffBase{src: remote}
-	version := make([]byte, 1)
-	if _, err = io.ReadFull(req.Body, version); err != nil {
-		Log.Debug("files::save::rdiff action=envelope_version err=%s", err.Error())
-		SendErrorResult(res, ErrNotValid)
-		return
-	} else if version[0] != 1 {
-		Log.Debug("files::save::rdiff action=envelope_version err=unsupported version=%d", version[0])
-		SendErrorResult(res, ErrNotImplemented)
-		return
-	}
-
 	root, filename := SplitPath(path)
 	part := root + "." + filename + ".part_" + QuickString(8)
 	hasher := sha256.New()
@@ -314,12 +298,9 @@ func handlerRDIFF(ctx *App, res http.ResponseWriter, req *http.Request, path str
 		Log.Debug("files::save::rdiff action=patch err=%s", err.Error())
 		SendErrorResult(res, NewError(err.Error(), status))
 	}
-	if err = librsync.Patch(io.NewSectionReader(base, 0, math.MaxInt64), req.Body, io.MultiWriter(writer, hasher)); err != nil {
+	if err = rdiffPatch(remote, req.Body, io.MultiWriter(writer, hasher)); err != nil {
 		abort(err, 403)
 		return
-	} else if base.err != nil {
-		abort(base.err, 403)
-		return
 	}
 	expected := make([]byte, sha256.Size)
 	if _, err = io.ReadFull(req.Body, expected); err != nil {
@@ -357,27 +338,6 @@ func handlerRDIFF(ctx *App, res http.ResponseWriter, req *http.Request, path str
 	SendSuccessResult(res, nil)
 }
 
-type rdiffBase struct {
-	src io.Reader
-	pos int64
-	err error
-}
-
-func (this *rdiffBase) ReadAt(p []byte, off int64) (int, error) {
-	if this.err != nil {
-		return 0, this.err
-	} else if off < this.pos {
-		this.err = errors.New("non monotonic access to the base file")
-		return 0, this.err
-	} else if _, err := io.CopyN(io.Discard, this.src, off-this.pos); err != nil {
-		this.err = err
-		return 0, err
-	}
-	n, err := io.ReadFull(this.src, p)
-	this.pos = off + int64(n)
-	return n, err
-}
-
 func createChunkedUploader(save func(path string, file io.Reader) error, path string, size uint64) *chunkedUpload {
 	r, w := io.Pipe()
 	done := make(chan error, 1)

server/pkg/files/utils_rdiff.go

@@ -0,0 +1,43 @@
+package files
+
+import (
+	"errors"
+	"io"
+	"math"
+
+	"github.com/balena-os/librsync-go"
+)
+
+func rdiffSignature(input io.Reader, output io.Writer) error {
+	_, err := librsync.Signature(input, output, 4096, 16, librsync.MD4_SIG_MAGIC)
+	return err
+}
+
+func rdiffPatch(remote io.Reader, delta io.Reader, output io.Writer) error {
+	base := &rdiffBase{src: remote}
+	if err := librsync.Patch(io.NewSectionReader(base, 0, math.MaxInt64), delta, output); err != nil {
+		return err
+	}
+	return base.err
+}
+
+type rdiffBase struct {
+	src io.Reader
+	pos int64
+	err error
+}
+
+func (this *rdiffBase) ReadAt(p []byte, off int64) (int, error) {
+	if this.err != nil {
+		return 0, this.err
+	} else if off < this.pos {
+		this.err = errors.New("non monotonic access to the base file")
+		return 0, this.err
+	} else if _, err := io.CopyN(io.Discard, this.src, off-this.pos); err != nil {
+		this.err = err
+		return 0, err
+	}
+	n, err := io.ReadFull(this.src, p)
+	this.pos = off + int64(n)
+	return n, err
+}