github/filestash
4
0
Fork 0
mirror of https://github.com/mickael-kerjean/filestash.git synced 2026-10-19 16:14:39 +00:00
Code Issues Releases Wiki Activity

feature (tracing): instrument storage call

Browse Source
This commit is contained in:
MickaelK
2026-05-12 21:57:43 +10:00
parent 9253a375ae
commit 543b6573c4
9 changed files with 98 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/common/default.go
+31 -16
View File
@@ -6,6 +6,8 @@ import (
"net"
"net/http"
"time"
"github.com/mickael-kerjean/filestash/server/pkg/tracer"
)
var USER_AGENT = fmt.Sprintf("Filestash/%s.%s (http://filestash.app)", APP_VERSION, BUILD_DATE)
@@ -16,32 +18,45 @@ func init() {
}
}
type HTTPClientOption func(*http.Transport)
func WithInsecure(t *http.Transport) {
t.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
type httpClientConfig struct {
transport *http.Transport
traceService string
}
func WithoutTimeout(t *http.Transport) {
t.ResponseHeaderTimeout = 0
type HTTPClientOption func(*httpClientConfig)
func WithInsecure(c *httpClientConfig) {
c.transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
func WithoutTimeout(c *httpClientConfig) {
c.transport.ResponseHeaderTimeout = 0
}
func WithTrace(service string) HTTPClientOption {
return func(c *httpClientConfig) {
c.traceService = service
}
}
func HTTPClient(opts ...HTTPClientOption) *http.Client {
t := &http.Transport{
Dial: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 10 * time.Second,
}).Dial,
TLSHandshakeTimeout: 5 * time.Second,
IdleConnTimeout: 60 * time.Second,
ResponseHeaderTimeout: 60 * time.Second,
cfg := &httpClientConfig{
transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 10 * time.Second,
}).Dial,
TLSHandshakeTimeout: 5 * time.Second,
IdleConnTimeout: 60 * time.Second,
ResponseHeaderTimeout: 60 * time.Second,
},
}
for _, opt := range opts {
opt(t)
opt(cfg)
}
return &http.Client{
Timeout: 5 * time.Hour,
Transport: NewTransformedTransport(t),
Transport: tracer.NewTransport(cfg.traceService, NewTransformedTransport(cfg.transport)),
}
}
server/pkg/tracer/transport.go
+38
View File
@@ -0,0 +1,38 @@
package tracer
import (
"fmt"
"net/http"
)
func NewTransport(svc string, orig http.RoundTripper) http.RoundTripper {
if svc == "" {
return orig
}
return &transport{svc, orig}
}
type transport struct {
svc string
orig http.RoundTripper
}
func (this *transport) RoundTrip(req *http.Request) (*http.Response, error) {
span := StartSpan(TraceFromContext(req.Context()), req.Method+" "+req.URL.Path, SpanOptions{
Kind: KindClient,
Service: this.svc,
Attributes: map[string]string{
"http.method": req.Method,
"http.url": req.URL.String(),
"http.host": req.URL.Host,
},
})
defer span.Close()
resp, err := this.orig.RoundTrip(req)
if err != nil {
span.SetError(err)
} else if resp.StatusCode >= 400 {
span.SetError(fmt.Errorf("status %d", resp.StatusCode))
}
return resp, err
}
server/plugin/plg_backend_artifactory/index.go
+9 -7
View File
@@ -21,6 +21,7 @@ func init() {
type ArtifactoryStorage struct {
instance string
token string
client *http.Client
}
func (this ArtifactoryStorage) Init(params map[string]string, app *App) (IBackend, error) {
@@ -30,6 +31,7 @@ func (this ArtifactoryStorage) Init(params map[string]string, app *App) (IBacken
}
this.token = params["token"]
this.instance = strings.TrimSuffix(strings.TrimSuffix(params["instance"], "/"), "/artifactory")
this.client = HTTPClient(WithTrace("artifactory"))
return this, nil
}
@@ -157,7 +159,7 @@ func (this ArtifactoryStorage) Ls(path string) ([]os.FileInfo, error) {
return nil, err
}
req.Header.Add("Authorization", "Bearer "+this.token)
res, err := HTTPClient().Do(req)
res, err := this.client.Do(req)
if err != nil {
return nil, err
}
@@ -189,7 +191,7 @@ func (this ArtifactoryStorage) Cat(path string) (io.ReadCloser, error) {
return nil, err
}
req.Header.Add("Authorization", "Bearer "+this.token)
res, err := HTTPClient().Do(req)
res, err := this.client.Do(req)
if err != nil {
return nil, err
}
@@ -231,7 +233,7 @@ func (this ArtifactoryStorage) Cat(path string) (io.ReadCloser, error) {
}
req, err = http.NewRequest("GET", artifactoryResponse.DownloadLink, nil)
req.Header.Add("Authorization", "Bearer "+this.token)
res, err = HTTPClient().Do(req)
res, err = this.client.Do(req)
return res.Body, err
}
@@ -268,7 +270,7 @@ func (this ArtifactoryStorage) Mkdir(path string) error {
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Authorization", "Bearer "+this.token)
res, err := HTTPClient().Do(req)
res, err := this.client.Do(req)
if err != nil {
return err
}
@@ -306,7 +308,7 @@ func (this ArtifactoryStorage) Rm(path string) error {
return err
}
req.Header.Add("Authorization", "Bearer "+this.token)
res, err := HTTPClient().Do(req)
res, err := this.client.Do(req)
if err != nil {
return err
}
@@ -335,7 +337,7 @@ func (this ArtifactoryStorage) Mv(from, to string) error {
return err
}
req.Header.Add("Authorization", "Bearer "+this.token)
res, err := HTTPClient().Do(req)
res, err := this.client.Do(req)
if err != nil {
return err
}
@@ -366,7 +368,7 @@ func (this ArtifactoryStorage) Save(path string, content io.Reader) error {
return err
}
req.Header.Add("Authorization", "Bearer "+this.token)
res, err := HTTPClient().Do(req)
res, err := this.client.Do(req)
if err != nil {
return err
}
server/plugin/plg_backend_backblaze/index.go
+3 -1
View File
@@ -28,6 +28,7 @@ type Backblaze struct {
AccountId string `json:"accountId"`
Token string `json:"authorizationToken"`
Status int `json:"status"`
client *http.Client
}
type BackblazeError struct {
@@ -43,6 +44,7 @@ func init() {
func (this Backblaze) Init(params map[string]string, app *App) (IBackend, error) {
this.params = params
this.client = HTTPClient(WithTrace("backblaze"))
// By default backblaze required quite a few API calls to just find the data that's under a given bucket
// This would result in a slow application hence we are caching everyting that's in the hot path
@@ -526,7 +528,7 @@ func (this Backblaze) request(method string, url string, body io.Reader, fn func
defer req.Body.Close()
}
res, err := HTTPClient().Do(req)
res, err := this.client.Do(req)
if err != nil {
return res, err
}
server/plugin/plg_backend_ipfs/index.go
+1 -1
View File
@@ -31,7 +31,7 @@ func (this Ipfs) Init(params map[string]string, app *App) (IBackend, error) {
if _, err := this.Stat("/"); err != nil {
return nil, err
}
this.client = HTTPClient()
this.client = HTTPClient(WithTrace("ipfs"))
return &this, nil
}
server/plugin/plg_backend_nop/index.go
+5 -1
View File
@@ -4,6 +4,7 @@ import (
. "github.com/mickael-kerjean/filestash/server/common"
"io"
"os"
"path/filepath"
"strconv"
"strings"
)
@@ -69,7 +70,10 @@ func (this BlackHole) Ls(path string) ([]os.FileInfo, error) {
}
func (this BlackHole) Stat(path string) (os.FileInfo, error) {
return nil, ErrNotImplemented
if path == "/" {
return File{FType: "directory"}, nil
}
return File{FName: filepath.Base(path), FType: "file"}, nil
}
func (this BlackHole) Cat(path string) (io.ReadCloser, error) {
server/plugin/plg_backend_perkeep/index.go
+4 -2
View File
@@ -18,6 +18,7 @@ func init() {
type Perkeep struct {
serverURL string
client *http.Client
}
func (this Perkeep) Init(params map[string]string, app *App) (IBackend, error) {
@@ -30,6 +31,7 @@ func (this Perkeep) Init(params map[string]string, app *App) (IBackend, error) {
}
return &Perkeep{
serverURL: url,
client: HTTPClient(WithTrace("perkeep")),
}, nil
}
@@ -220,7 +222,7 @@ func (this *Perkeep) query(searchRequest any) (*SearchResponse, error) {
if err != nil {
return nil, NewError("Failed to create request: "+err.Error(), 500)
}
resp, err := HTTPClient().Do(req)
resp, err := this.client.Do(req)
if err != nil {
return nil, NewError("Failed to query perkeep: "+err.Error(), 500)
}
@@ -249,7 +251,7 @@ func (this *Perkeep) describe(blobRef string) (*DescribeResponse, error) {
if err != nil {
return nil, NewError("Failed to create request: "+err.Error(), 500)
}
resp, err := HTTPClient().Do(req)
resp, err := this.client.Do(req)
if err != nil {
return nil, NewError("Failed to describe perkeep: "+err.Error(), 500)
}
server/plugin/plg_backend_url/index.go
+1 -1
View File
@@ -51,7 +51,7 @@ func (this Url) Init(params map[string]string, app *App) (IBackend, error) {
}
home := u.Path
u.Path = "/"
opts := []HTTPClientOption{}
opts := []HTTPClientOption{WithTrace("url")}
if params["insecure"] == "on" {
opts = append(opts, WithInsecure)
}
server/plugin/plg_backend_webdav/index.go
+6 -1
View File
@@ -16,6 +16,8 @@ import (
type WebDav struct {
params *WebDavParams
client *http.Client
app *App
}
type WebDavParams struct {
@@ -41,6 +43,8 @@ func (w WebDav) Init(params map[string]string, app *App) (IBackend, error) {
password: params["password"],
path: params["path"],
},
client: HTTPClient(WithTrace("webdav")),
app: app,
}
return backend, nil
}
@@ -265,6 +269,7 @@ func (w WebDav) request(method string, url string, body io.Reader, fn func(req *
if err != nil {
return nil, err
}
req = req.WithContext(w.app.Context)
if w.params.username != "" {
req.SetBasicAuth(w.params.username, w.params.password)
}
@@ -284,7 +289,7 @@ func (w WebDav) request(method string, url string, body io.Reader, fn func(req *
if fn != nil {
fn(req)
}
return HTTPClient().Do(req)
return w.client.Do(req)
}
type WebDavResp struct {