github/filestash
4
0
Fork 0
mirror of https://github.com/mickael-kerjean/filestash.git synced 2026-10-19 16:14:39 +00:00
Code Issues Releases Wiki Activity

feature (optimisation): faster hot path & reduce allocs

Browse Source
This commit is contained in:
MickaelK
2026-08-19 16:34:15 +10:00
parent 4271a6a86c
commit 4414aec58a
6 changed files with 193 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/common/log.go
+46 -35
View File
@@ -1,25 +1,34 @@
package common
import (
"bytes"
"fmt"
"io"
slog "log"
"os"
"strings"
"sync"
"time"
)
var (
Log = &log{enable: true}
logfile *os.File
)
var Log = &log{
enable: true,
out: [2]io.Writer{io.Discard, io.Discard},
buf: sync.Pool{
New: func() any {
return new(bytes.Buffer)
},
},
}
func InitLogger() (err error) {
logfile, err = os.OpenFile(GetAbsolutePath(LOG_PATH, "access.log"), os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm)
f, err := os.OpenFile(GetAbsolutePath(LOG_PATH, "access.log"), os.O_APPEND|os.O_WRONLY|os.O_CREATE, os.ModePerm)
if err != nil {
slog.Printf("ERROR log file: %+v", err)
return err
}
logfile.WriteString("")
f.WriteString("")
Log.out[0] = f
Log.out[1] = os.Stdout
return nil
}
@@ -29,62 +38,64 @@ type log struct {
info bool
warn bool
error bool
out [2]io.Writer
buf sync.Pool
}
func (l *log) Info(format string, v ...interface{}) {
if l.info && l.enable {
message := fmt.Sprintf("%s SYST INFO ", l.now())
message = fmt.Sprintf(message+format+"\n", v...)
logfile.WriteString(message)
fmt.Print(strings.Replace(message, "%", "%%", -1))
l.Stdout("SYST INFO "+format, v...)
}
}
func (l *log) Warning(format string, v ...interface{}) {
if l.warn && l.enable {
message := fmt.Sprintf("%s SYST WARN ", l.now())
message = fmt.Sprintf(message+format+"\n", v...)
logfile.WriteString(message)
fmt.Print(strings.Replace(message, "%", "%%", -1))
l.Stdout("SYST WARN "+format, v...)
}
}
func (l *log) Error(format string, v ...interface{}) {
if l.error && l.enable {
message := fmt.Sprintf("%s SYST ERROR ", l.now())
message = fmt.Sprintf(message+format+"\n", v...)
logfile.WriteString(message)
fmt.Print(strings.Replace(message, "%", "%%", -1))
l.Stdout("SYST ERROR "+format, v...)
}
}
func (l *log) Debug(format string, v ...interface{}) {
if l.debug && l.enable {
message := fmt.Sprintf("%s SYST DEBUG ", l.now())
message = fmt.Sprintf(message+format+"\n", v...)
logfile.WriteString(message)
fmt.Print(strings.Replace(message, "%", "%%", -1))
l.Stdout("SYST DEBUG "+format, v...)
}
}
func (l *log) Stdout(format string, v ...interface{}) {
message := fmt.Sprintf("%s ", l.now())
message = fmt.Sprintf(message+format+"\n", v...)
logfile.WriteString(message)
fmt.Print(strings.Replace(message, "%", "%%", -1))
buf := l.buf.Get().(*bytes.Buffer)
defer l.buf.Put(buf)
buf.Reset()
fmt.Fprintf(buf, format, v...)
l.Raw(buf.Bytes())
}
func (l *log) now() string {
return time.Now().Format("2006/01/02 15:04:05")
func (l *log) Raw(body []byte) {
buf := l.buf.Get().(*bytes.Buffer)
defer l.buf.Put(buf)
buf.Reset()
var ts [20]byte
buf.Write(time.Now().AppendFormat(ts[:0], "2006/01/02 15:04:05"))
buf.WriteByte(' ')
buf.Write(body)
buf.WriteByte('\n')
b := buf.Bytes()
for _, w := range l.out {
w.Write(b)
}
}
func (l *log) Close() {
logfile.Close()
for _, w := range l.out {
if c, ok := w.(io.Closer); ok {
c.Close()
}
}
}
func (l *log) SetVisibility(str string) {
server/common/mime.go
+1 -2
View File
@@ -13,8 +13,7 @@ func GetMimeType(p string) string {
if ext != "" {
ext = ext[1:]
}
ext = strings.ToLower(ext)
mType := MimeTypes[ext]
mType := MimeTypes[strings.ToLower(ext)]
if mType == "" {
return "application/octet-stream"
}
server/middleware/http.go
+44 -23
View File
@@ -12,33 +12,43 @@ import (
)
func ApiHeaders(fn HandlerFunc) HandlerFunc {
var (
headerContentType = []string{"application/json"}
headerContentCache = []string{"no-cache"}
)
return HandlerFunc(func(ctx *App, res http.ResponseWriter, req *http.Request) {
header := res.Header()
header.Set("Content-Type", "application/json")
header.Set("Cache-Control", "no-cache")
if id := req.Header.Get("X-Request-ID"); id != "" {
header.Set("X-Request-ID", id)
header["Content-Type"] = headerContentType
header["Cache-Control"] = headerContentCache
if id := req.Header.Get("X-Request-Id"); id != "" {
header["X-Request-Id"] = []string{id}
}
fn(ctx, res, req)
})
}
func StaticHeaders(fn HandlerFunc) HandlerFunc {
var headerContentCache = []string{"max-age=2592000"}
return HandlerFunc(func(ctx *App, res http.ResponseWriter, req *http.Request) {
header := res.Header()
header.Set("Content-Type", GetMimeType(filepath.Ext(req.URL.Path)))
header.Set("Cache-Control", "max-age=2592000")
header["Cache-Control"] = headerContentCache
header["Content-Type"] = []string{GetMimeType(filepath.Ext(req.URL.Path))}
fn(ctx, res, req)
})
}
func PublicCORS(fn HandlerFunc) HandlerFunc {
var (
headerAccessControlAllowOrigin = []string{"*"}
headerAccessControlAllowHeaders = []string{"x-requested-with, x-request-id"}
headerAccessControlAllowMethods = []string{"GET, OPTIONS"}
)
return HandlerFunc(func(ctx *App, res http.ResponseWriter, req *http.Request) {
header := res.Header()
header.Set("Access-Control-Allow-Origin", "*")
header.Set("Access-Control-Allow-Headers", "x-requested-with, x-request-id")
header["Access-Control-Allow-Origin"] = headerAccessControlAllowOrigin
header["Access-Control-Allow-Headers"] = headerAccessControlAllowHeaders
if req.Method == http.MethodOptions {
header.Set("Access-Control-Allow-Methods", "GET, OPTIONS")
header["Access-Control-Allow-Methods"] = headerAccessControlAllowMethods
res.WriteHeader(http.StatusNoContent)
return
}
@@ -47,31 +57,45 @@ func PublicCORS(fn HandlerFunc) HandlerFunc {
}
func IndexHeaders(fn HandlerFunc) HandlerFunc {
var (
headerContentType = []string{"text/html"}
headerCacheControl = []string{"no-cache"}
headerReferrerPolicy = []string{"same-origin"}
headerXContentTypeOptions = []string{"nosniff"}
headerXXSSProtection = []string{"1; mode=block"}
headerXPoweredBy = []string{fmt.Sprintf("Filestash/%s.%s <https://filestash.app>", APP_VERSION, BUILD_DATE)}
headerXFrameOptions = []string{"DENY"}
)
return HandlerFunc(func(ctx *App, res http.ResponseWriter, req *http.Request) {
header := res.Header()
header.Set("Content-Type", "text/html")
header.Set("Cache-Control", "no-cache")
header.Set("Referrer-Policy", "same-origin")
header.Set("X-Content-Type-Options", "nosniff")
header.Set("X-XSS-Protection", "1; mode=block")
header["Content-Type"] = headerContentType
header["Cache-Control"] = headerCacheControl
header["Referrer-Policy"] = headerReferrerPolicy
header["X-Content-Type-Options"] = headerXContentTypeOptions
header["X-XSS-Protection"] = headerXXSSProtection
if !IsWhiteLabel() {
header.Set("X-Powered-By", fmt.Sprintf("Filestash/%s.%s <https://filestash.app>", APP_VERSION, BUILD_DATE))
header["X-Powered-By"] = headerXPoweredBy
}
if ori := Config.Get("features.protection.iframe").String(); ori == "" {
header.Set("X-Frame-Options", "DENY")
header["X-Frame-Options"] = headerXFrameOptions
}
fn(ctx, res, req)
})
}
func SecureHeaders(fn HandlerFunc) HandlerFunc {
var (
headerStrictTransportSecurity = []string{"max-age=31536000; includeSubDomains; preload"}
headerXContentTypeOptions = []string{"nosniff"}
headerXXSSProtection = []string{"1; mode=block"}
)
return HandlerFunc(func(ctx *App, res http.ResponseWriter, req *http.Request) {
header := res.Header()
if Config.Get("general.force_ssl").Bool() {
header.Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload")
header["Strict-Transport-Security"] = headerStrictTransportSecurity
}
header.Set("X-Content-Type-Options", "nosniff")
header.Set("X-XSS-Protection", "1; mode=block")
header["X-Content-Type-Options"] = headerXContentTypeOptions
header["X-XSS-Protection"] = headerXXSSProtection
fn(ctx, res, req)
})
}
@@ -110,10 +134,7 @@ func RateLimiter(fn HandlerFunc) HandlerFunc {
return HandlerFunc(func(ctx *App, res http.ResponseWriter, req *http.Request) {
if limiter.Allow() == false {
Log.Warning("middleware::http::ratelimit too many requests")
SendErrorResult(
res,
NewError(http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests),
)
SendErrorResult(res, NewError(http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests))
return
}
fn(ctx, res, req)
server/middleware/index.go
+11 -7
View File
@@ -3,6 +3,7 @@ package middleware
import (
"net/http"
"time"
_ "unsafe"
. "github.com/mickael-kerjean/filestash/server/common"
)
@@ -20,32 +21,32 @@ func init() {
func NewMiddlewareChain(fn HandlerFunc, m []Middleware) http.HandlerFunc {
return func(res http.ResponseWriter, req *http.Request) {
var resw ResponseWriter = NewResponseWriter(res)
var f func(*App, http.ResponseWriter, *http.Request) = fn
for i := len(m) - 1; i >= 0; i-- {
f = m[i](f)
}
app := App{
Context: req.Context(),
}
var (
app = App{Context: req.Context()}
resw = NewResponseWriter(res)
)
f(&app, &resw, req)
if req.Body != nil {
req.Body.Close()
}
go logger(&app, &resw, req)
logger(&app, &resw, req)
}
}
type ResponseWriter struct {
http.ResponseWriter
status int
start time.Time
start int64
}
func NewResponseWriter(res http.ResponseWriter) ResponseWriter {
return ResponseWriter{
ResponseWriter: res,
start: time.Now(),
start: now(),
}
}
@@ -75,3 +76,6 @@ func PluginInjector(fn HandlerFunc) HandlerFunc {
}
return fn
}
//go:linkname now runtime.nanotime
func now() int64
server/middleware/telemetry.go
+87 -75
View File
@@ -4,13 +4,16 @@ import (
"bytes"
"encoding/json"
"net/http"
"strconv"
"sync"
"time"
. "github.com/mickael-kerjean/filestash/server/common"
)
var telemetry = Telemetry{Data: make([]LogEntry, 0)}
var (
telemetry = Telemetry{Data: make([]LogEntry, 0)}
version = APP_VERSION + "." + BUILD_DATE
)
type Telemetry struct {
Data []LogEntry
@@ -18,88 +21,97 @@ type Telemetry struct {
}
type LogEntry struct {
Host string `json:"host"`
Method string `json:"method"`
RequestURI string `json:"pathname"`
Proto string `json:"proto"`
Status int `json:"status"`
Scheme string `json:"scheme"`
UserAgent string `json:"userAgent"`
Ip string `json:"ip"`
Referer string `json:"referer"`
Duration float64 `json:"responseTime"`
Version string `json:"version"`
Backend string `json:"backend"`
Share string `json:"share"`
License string `json:"license"`
Session string `json:"session"`
RequestID string `json:"requestID"`
Host string `json:"host"`
Method string `json:"method"`
RequestURI string `json:"pathname"`
Proto string `json:"proto"`
Status int `json:"status"`
Scheme string `json:"scheme"`
UserAgent string `json:"userAgent"`
Ip string `json:"ip"`
Referer string `json:"referer"`
Duration int `json:"responseTime"`
Version string `json:"version"`
Backend string `json:"backend"`
Share string `json:"share"`
License string `json:"license"`
Session string `json:"session"`
RequestID string `json:"requestID"`
}
func logger(ctx *App, res http.ResponseWriter, req *http.Request) {
if obj, ok := res.(*ResponseWriter); ok && req.RequestURI != "/about" {
point := LogEntry{
Version: APP_VERSION + "." + BUILD_DATE,
License: LICENSE,
Scheme: req.URL.Scheme,
Host: req.Host,
Method: req.Method,
RequestURI: req.RequestURI,
Proto: req.Proto,
Status: obj.status,
UserAgent: req.Header.Get("User-Agent"),
Ip: req.RemoteAddr,
Referer: req.Referer(),
Duration: float64(time.Now().Sub(obj.start)) / (1000 * 1000),
Backend: func() string {
if ctx.Session["type"] == "" {
return "null"
}
return ctx.Session["type"]
}(),
Share: func() string {
if ctx.Share.Id == "" {
return "null"
}
return ctx.Share.Id
}(),
Session: func() string {
if ctx.Session["type"] == "" {
return "null"
}
return GenerateID(ctx.Session)
}(),
RequestID: func() string {
defer func() string {
if r := recover(); r != nil {
return "oops"
}
return "null"
}()
return res.Header().Get("X-Request-ID")
}(),
}
if Config.Get("log.telemetry").Bool() {
telemetry.Record(point)
}
if Config.Get("log.enable").Bool() {
tid := ""
if point.RequestID != "" && Config.Get("log.level").String() == "DEBUG" {
tid = "trace=" + point.RequestID
func logger(ctx *App, res *ResponseWriter, req *http.Request) {
if req.RequestURI == "/about" {
return
}
point := LogEntry{
Version: version,
License: LICENSE,
Scheme: req.URL.Scheme,
Host: req.Host,
Method: req.Method,
RequestURI: req.RequestURI,
Proto: req.Proto,
Status: res.status,
UserAgent: req.Header.Get("User-Agent"),
Ip: req.RemoteAddr,
Referer: req.Referer(),
Duration: int((now() - res.start) / (100_000)),
Backend: func() string {
if ctx.Session["type"] == "" {
return "null"
}
Log.Stdout("HTTP %3d %3s %6.1fms %s %s", point.Status, point.Method, point.Duration, limit(point.RequestURI, 200), tid)
return ctx.Session["type"]
}(),
Share: func() string {
if ctx.Share.Id == "" {
return "null"
}
return ctx.Share.Id
}(),
Session: func() string {
if ctx.Session["type"] == "" {
return "null"
}
return GenerateID(ctx.Session)
}(),
RequestID: res.Header().Get("X-Request-Id"),
}
if Config.Get("log.telemetry").Bool() {
telemetry.Record(point)
}
if Config.Get("log.enable").Bool() {
var (
arr [512]byte
num [32]byte
)
buf := bytes.NewBuffer(arr[:0])
buf.WriteString("HTTP ")
buf.Write(strconv.AppendInt(num[:0], int64(point.Status), 10))
buf.WriteByte(' ')
buf.WriteString(point.Method)
buf.WriteByte(' ')
buf.Write(strconv.AppendInt(num[:0], int64(point.Duration/10), 10))
buf.WriteByte('.')
buf.Write(strconv.AppendInt(num[:0], int64(point.Duration%10), 10))
buf.WriteString("ms ")
if uri := point.RequestURI; len(uri) > 200 {
buf.WriteString(uri[:200])
buf.WriteString("...")
} else {
buf.WriteString(uri)
}
buf.WriteByte(' ')
if point.RequestID != "" && Config.Get("log.level").String() == "DEBUG" {
buf.WriteString("trace=")
buf.WriteString(point.RequestID)
}
Log.Raw(buf.Bytes())
}
}
func limit(input string, maxLength int) string {
if len(input) > maxLength {
return input[:maxLength] + "..."
}
return input
}
func (this *Telemetry) Record(point LogEntry) {
point.Duration *= 10
this.mu.Lock()
this.Data = append(this.Data, point)
this.mu.Unlock()
server/model/files.go
+4 -3
View File
@@ -10,9 +10,10 @@ func NewBackend(ctx *App, conn map[string]string) (IBackend, error) {
isAllowed := func() bool {
// by default, a hacker could use filestash to establish connections outside of what's
// define in the config file. We need to prevent this
connections := Config.Connections()
possibilities := make([]map[string]interface{}, 0)
for i := 0; i < len(Config.Conn); i++ {
d := Config.Conn[i]
for i := 0; i < len(connections); i++ {
d := connections[i]
if d["type"] != conn["type"] {
continue
}
@@ -36,7 +37,7 @@ func NewBackend(ctx *App, conn map[string]string) (IBackend, error) {
continue
}
}
possibilities = append(possibilities, Config.Conn[i])
possibilities = append(possibilities, connections[i])
}
if len(possibilities) > 0 {
return true