logs: add virtual path to transfer/command logs and event CSV columns

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

internal/common/connection.go

@@ -418,7 +418,7 @@ func (c *BaseConnection) CreateDir(virtualPath string, checkFilePatterns bool) e
 	vfs.SetPathPermissions(fs, fsPath, c.User.GetUID(), c.User.GetGID())
 	elapsed := time.Since(startTime).Nanoseconds() / 1000000
 
-	logger.CommandLog(mkdirLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1,
+	logger.CommandLog(mkdirLogSender, fsPath, "", virtualPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1,
 		c.localAddr, c.remoteAddr, elapsed)
 	ExecuteActionNotification(c, operationMkdir, fsPath, virtualPath, "", "", "", 0, nil, elapsed, nil) //nolint:errcheck
 	return nil
@@ -462,7 +462,7 @@ func (c *BaseConnection) RemoveFile(fs vfs.Fs, fsPath, virtualPath string, info
 	}
 	elapsed := time.Since(startTime).Nanoseconds() / 1000000
 
-	logger.CommandLog(removeLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1,
+	logger.CommandLog(removeLogSender, fsPath, "", virtualPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1,
 		c.localAddr, c.remoteAddr, elapsed)
 	if updateQuota && info.Mode()&os.ModeSymlink == 0 {
 		vfolder, err := c.User.GetVirtualFolderForPath(path.Dir(virtualPath))
@@ -536,7 +536,7 @@ func (c *BaseConnection) RemoveDir(virtualPath string) error {
 	}
 	elapsed := time.Since(startTime).Nanoseconds() / 1000000
 
-	logger.CommandLog(rmdirLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1,
+	logger.CommandLog(rmdirLogSender, fsPath, "", virtualPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1,
 		c.localAddr, c.remoteAddr, elapsed)
 	ExecuteActionNotification(c, operationRmdir, fsPath, virtualPath, "", "", "", 0, nil, elapsed, nil) //nolint:errcheck
 	return nil
@@ -660,8 +660,9 @@ func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, s
 			numFiles, sizeDiff, err := copier.CopyFile(fsSourcePath, fsTargetPath, srcInfo)
 			elapsed := time.Since(startTime).Nanoseconds() / 1000000
 			updateUserQuotaAfterFileWrite(c, virtualTargetPath, numFiles, sizeDiff)
-			logger.CommandLog(copyLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1,
-				"", "", "", srcInfo.Size(), c.localAddr, c.remoteAddr, elapsed)
+			logger.CommandLog(copyLogSender, fsSourcePath, fsTargetPath, virtualSourcePath, virtualTargetPath,
+				c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", srcInfo.Size(),
+				c.localAddr, c.remoteAddr, elapsed)
 			ExecuteActionNotification(c, operationCopy, fsSourcePath, virtualSourcePath, fsTargetPath, virtualTargetPath, "", srcInfo.Size(), err, elapsed, nil) //nolint:errcheck
 			return err
 		}
@@ -872,8 +873,8 @@ func (c *BaseConnection) renameInternal(virtualSourcePath, virtualTargetPath str
 	vfs.SetPathPermissions(fsDst, fsTargetPath, c.User.GetUID(), c.User.GetGID())
 	elapsed := time.Since(startTime).Nanoseconds() / 1000000
 	c.updateQuotaAfterRename(fsDst, virtualSourcePath, virtualTargetPath, fsTargetPath, initialSize, files, size) //nolint:errcheck
-	logger.CommandLog(renameLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1,
-		"", "", "", -1, c.localAddr, c.remoteAddr, elapsed)
+	logger.CommandLog(renameLogSender, fsSourcePath, fsTargetPath, virtualSourcePath, virtualTargetPath,
+		c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1, c.localAddr, c.remoteAddr, elapsed)
 	ExecuteActionNotification(c, operationRename, fsSourcePath, virtualSourcePath, fsTargetPath, //nolint:errcheck
 		virtualTargetPath, "", 0, nil, elapsed, nil)
 
@@ -931,8 +932,8 @@ func (c *BaseConnection) CreateSymlink(virtualSourcePath, virtualTargetPath stri
 		return c.GetFsError(fs, err)
 	}
 	elapsed := time.Since(startTime).Nanoseconds() / 1000000
-	logger.CommandLog(symlinkLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, "",
-		"", "", -1, c.localAddr, c.remoteAddr, elapsed)
+	logger.CommandLog(symlinkLogSender, fsSourcePath, fsTargetPath, virtualSourcePath, virtualTargetPath,
+		c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "", -1, c.localAddr, c.remoteAddr, elapsed)
 	return nil
 }
 
@@ -1012,7 +1013,7 @@ func (c *BaseConnection) ignoreSetStat(fs vfs.Fs) bool {
 	return false
 }
 
-func (c *BaseConnection) handleChmod(fs vfs.Fs, fsPath, pathForPerms string, attributes *StatAttributes) error {
+func (c *BaseConnection) handleChmod(fs vfs.Fs, fsPath, virtualPath, pathForPerms string, attributes *StatAttributes) error {
 	if !c.User.HasPerm(dataprovider.PermChmod, pathForPerms) {
 		return c.GetPermissionDeniedError()
 	}
@@ -1025,12 +1026,12 @@ func (c *BaseConnection) handleChmod(fs vfs.Fs, fsPath, pathForPerms string, att
 		return c.GetFsError(fs, err)
 	}
 	elapsed := time.Since(startTime).Nanoseconds() / 1000000
-	logger.CommandLog(chmodLogSender, fsPath, "", c.User.Username, attributes.Mode.String(), c.ID, c.protocol,
-		-1, -1, "", "", "", -1, c.localAddr, c.remoteAddr, elapsed)
+	logger.CommandLog(chmodLogSender, fsPath, "", virtualPath, "", c.User.Username, attributes.Mode.String(),
+		c.ID, c.protocol, -1, -1, "", "", "", -1, c.localAddr, c.remoteAddr, elapsed)
 	return nil
 }
 
-func (c *BaseConnection) handleChown(fs vfs.Fs, fsPath, pathForPerms string, attributes *StatAttributes) error {
+func (c *BaseConnection) handleChown(fs vfs.Fs, fsPath, virtualPath, pathForPerms string, attributes *StatAttributes) error {
 	if !c.User.HasPerm(dataprovider.PermChown, pathForPerms) {
 		return c.GetPermissionDeniedError()
 	}
@@ -1044,12 +1045,12 @@ func (c *BaseConnection) handleChown(fs vfs.Fs, fsPath, pathForPerms string, att
 		return c.GetFsError(fs, err)
 	}
 	elapsed := time.Since(startTime).Nanoseconds() / 1000000
-	logger.CommandLog(chownLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, attributes.UID, attributes.GID,
-		"", "", "", -1, c.localAddr, c.remoteAddr, elapsed)
+	logger.CommandLog(chownLogSender, fsPath, "", virtualPath, "", c.User.Username, "", c.ID, c.protocol,
+		attributes.UID, attributes.GID, "", "", "", -1, c.localAddr, c.remoteAddr, elapsed)
 	return nil
 }
 
-func (c *BaseConnection) handleChtimes(fs vfs.Fs, fsPath, pathForPerms string, attributes *StatAttributes) error {
+func (c *BaseConnection) handleChtimes(fs vfs.Fs, fsPath, virtualPath, pathForPerms string, attributes *StatAttributes) error {
 	if !c.User.HasPerm(dataprovider.PermChtimes, pathForPerms) {
 		return c.GetPermissionDeniedError()
 	}
@@ -1070,7 +1071,7 @@ func (c *BaseConnection) handleChtimes(fs vfs.Fs, fsPath, pathForPerms string, a
 	elapsed := time.Since(startTime).Nanoseconds() / 1000000
 	accessTimeString := attributes.Atime.Format(chtimesFormat)
 	modificationTimeString := attributes.Mtime.Format(chtimesFormat)
-	logger.CommandLog(chtimesLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1,
+	logger.CommandLog(chtimesLogSender, fsPath, "", virtualPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1,
 		accessTimeString, modificationTimeString, "", -1, c.localAddr, c.remoteAddr, elapsed)
 	return nil
 }
@@ -1087,19 +1088,19 @@ func (c *BaseConnection) SetStat(virtualPath string, attributes *StatAttributes)
 	pathForPerms := path.Dir(virtualPath)
 
 	if attributes.Flags&StatAttrTimes != 0 {
-		if err = c.handleChtimes(fs, fsPath, pathForPerms, attributes); err != nil {
+		if err = c.handleChtimes(fs, fsPath, virtualPath, pathForPerms, attributes); err != nil {
 			return err
 		}
 	}
 
 	if attributes.Flags&StatAttrPerms != 0 {
-		if err = c.handleChmod(fs, fsPath, pathForPerms, attributes); err != nil {
+		if err = c.handleChmod(fs, fsPath, virtualPath, pathForPerms, attributes); err != nil {
 			return err
 		}
 	}
 
 	if attributes.Flags&StatAttrUIDGID != 0 {
-		if err = c.handleChown(fs, fsPath, pathForPerms, attributes); err != nil {
+		if err = c.handleChown(fs, fsPath, virtualPath, pathForPerms, attributes); err != nil {
 			return err
 		}
 	}
@@ -1114,8 +1115,8 @@ func (c *BaseConnection) SetStat(virtualPath string, attributes *StatAttributes)
 			return c.GetFsError(fs, err)
 		}
 		elapsed := time.Since(startTime).Nanoseconds() / 1000000
-		logger.CommandLog(truncateLogSender, fsPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "",
-			"", attributes.Size, c.localAddr, c.remoteAddr, elapsed)
+		logger.CommandLog(truncateLogSender, fsPath, "", virtualPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1,
+			"", "", "", attributes.Size, c.localAddr, c.remoteAddr, elapsed)
 	}
 
 	return nil

internal/common/connection_test.go

@@ -144,17 +144,17 @@ func TestSetStatMode(t *testing.T) {
 	user.Permissions["/"] = []string{dataprovider.PermAny}
 	fs := newMockOsFs(true, "", user.GetHomeDir(), "", nil)
 	conn := NewBaseConnection("", ProtocolWebDAV, "", "", user)
-	err := conn.handleChmod(fs, fakePath, fakePath, nil)
+	err := conn.handleChmod(fs, fakePath, fakePath, fakePath, nil)
 	assert.NoError(t, err)
-	err = conn.handleChown(fs, fakePath, fakePath, nil)
+	err = conn.handleChown(fs, fakePath, fakePath, fakePath, nil)
 	assert.NoError(t, err)
-	err = conn.handleChtimes(fs, fakePath, fakePath, nil)
+	err = conn.handleChtimes(fs, fakePath, fakePath, fakePath, nil)
 	assert.NoError(t, err)
 
 	Config.SetstatMode = 2
-	err = conn.handleChmod(fs, fakePath, fakePath, nil)
+	err = conn.handleChmod(fs, fakePath, fakePath, fakePath, nil)
 	assert.NoError(t, err)
-	err = conn.handleChtimes(fs, fakePath, fakePath, &StatAttributes{
+	err = conn.handleChtimes(fs, fakePath, fakePath, fakePath, &StatAttributes{
 		Atime: time.Now(),
 		Mtime: time.Now(),
 	})

internal/common/eventmanager.go

@@ -923,8 +923,9 @@ func closeWriterAndUpdateQuota(w io.WriteCloser, conn *BaseConnection, virtualSo
 				errTransfer = errWrite
 			}
 			if operation == operationCopy {
-				logger.CommandLog(copyLogSender, fsSrcPath, fsDstPath, conn.User.Username, "", conn.ID, conn.protocol, -1, -1,
-					"", "", "", info.Size(), conn.localAddr, conn.remoteAddr, elapsed)
+				logger.CommandLog(copyLogSender, fsSrcPath, fsDstPath, virtualSourcePath, virtualTargetPath,
+					conn.User.Username, "", conn.ID, conn.protocol, -1, -1, "", "", "", info.Size(),
+					conn.localAddr, conn.remoteAddr, elapsed)
 			}
 			ExecuteActionNotification(conn, operation, fsSrcPath, virtualSourcePath, fsDstPath, virtualTargetPath, "", info.Size(), errTransfer, elapsed, nil) //nolint:errcheck
 		}

internal/common/transfer.go

@@ -427,7 +427,7 @@ func (t *BaseTransfer) Close() error {
 	elapsed := time.Since(t.start).Nanoseconds() / 1000000
 	var uploadFileSize int64
 	if t.transferType == TransferDownload {
-		logger.TransferLog(downloadLogSender, t.fsPath, elapsed, t.BytesSent.Load(), t.Connection.User.Username,
+		logger.TransferLog(downloadLogSender, t.fsPath, t.requestPath, elapsed, t.BytesSent.Load(), t.Connection.User.Username,
 			t.Connection.ID, t.Connection.protocol, t.Connection.localAddr, t.Connection.remoteAddr, t.ftpMode,
 			t.ErrTransfer)
 		ExecuteActionNotification(t.Connection, operationDownload, t.fsPath, t.requestPath, "", "", "", //nolint:errcheck
@@ -449,7 +449,7 @@ func (t *BaseTransfer) Close() error {
 		numFiles, uploadFileSize = t.executeUploadHook(numFiles, uploadFileSize, elapsed)
 		t.updateQuota(numFiles, uploadFileSize)
 		t.updateTimes()
-		logger.TransferLog(uploadLogSender, t.fsPath, elapsed, t.BytesReceived.Load(), t.Connection.User.Username,
+		logger.TransferLog(uploadLogSender, t.fsPath, t.requestPath, elapsed, t.BytesReceived.Load(), t.Connection.User.Username,
 			t.Connection.ID, t.Connection.protocol, t.Connection.localAddr, t.Connection.remoteAddr, t.ftpMode,
 			t.ErrTransfer)
 	}

internal/httpd/api_events.go

@@ -382,22 +382,27 @@ type fsEvent struct {
 	Endpoint          string `json:"endpoint,omitempty"`
 	OpenFlags         int    `json:"open_flags,omitempty"`
 	Role              string `json:"role,omitempty"`
-	InstanceID        string `json:"instance_id,omitempty"`
 }
 
 func (e *fsEvent) getCSVHeader() []string {
-	return []string{"Time", "Action", "Path", "Size", "Elapsed", "Status", "User", "Protocol",
-		"IP", "SSH command"}
+	return []string{"Time", "Action", "Path", "Fs Path", "Size", "Elapsed", "Status", "User", "Protocol",
+		"IP", "SSH command", "Role"}
 }
 
 func (e *fsEvent) getCSVData() []string {
 	timestamp := time.Unix(0, e.Timestamp).UTC()
 	var pathInfo strings.Builder
-	pathInfo.Write([]byte(e.VirtualPath))
+	pathInfo.WriteString(e.VirtualPath)
 	if e.VirtualTargetPath != "" {
 		pathInfo.WriteString(" => ")
 		pathInfo.WriteString(e.VirtualTargetPath)
 	}
+	var fsPathInfo strings.Builder
+	fsPathInfo.WriteString(e.FsPath)
+	if e.FsTargetPath != "" {
+		fsPathInfo.WriteString(" => ")
+		fsPathInfo.WriteString(e.FsTargetPath)
+	}
 	var status string
 	switch e.Status {
 	case 1:
@@ -415,8 +420,8 @@ func (e *fsEvent) getCSVData() []string {
 	if e.Elapsed > 0 {
 		elapsed = (time.Duration(e.Elapsed) * time.Millisecond).String()
 	}
-	return []string{timestamp.Format(time.RFC3339Nano), e.Action, pathInfo.String(),
-		fileSize, elapsed, status, e.Username, e.Protocol, e.IP, e.SSHCmd}
+	return []string{timestamp.Format(time.RFC3339Nano), e.Action, pathInfo.String(), fsPathInfo.String(),
+		fileSize, elapsed, status, e.Username, e.Protocol, e.IP, e.SSHCmd, e.Role}
 }
 
 type providerEvent struct {
@@ -429,17 +434,16 @@ type providerEvent struct {
 	ObjectName string `json:"object_name"`
 	ObjectData []byte `json:"object_data"`
 	Role       string `json:"role,omitempty"`
-	InstanceID string `json:"instance_id,omitempty"`
 }
 
 func (e *providerEvent) getCSVHeader() []string {
-	return []string{"Time", "Action", "Object Type", "Object Name", "User", "IP"}
+	return []string{"Time", "Action", "Object Type", "Object Name", "User", "IP", "Role"}
 }
 
 func (e *providerEvent) getCSVData() []string {
 	timestamp := time.Unix(0, e.Timestamp).UTC()
 	return []string{timestamp.Format(time.RFC3339Nano), e.Action, e.ObjectType, e.ObjectName,
-		e.Username, e.IP}
+		e.Username, e.IP, e.Role}
 }
 
 type logEvent struct {
@@ -454,13 +458,13 @@ type logEvent struct {
 }
 
 func (e *logEvent) getCSVHeader() []string {
-	return []string{"Time", "Event", "Protocol", "User", "IP", "Message"}
+	return []string{"Time", "Event", "Protocol", "User", "IP", "Message", "Role"}
 }
 
 func (e *logEvent) getCSVData() []string {
 	timestamp := time.Unix(0, e.Timestamp).UTC()
 	return []string{timestamp.Format(time.RFC3339Nano), getLogEventString(notifier.LogEventType(e.Event)),
-		e.Protocol, e.Username, e.IP, e.Message}
+		e.Protocol, e.Username, e.IP, e.Message, e.Role}
 }
 
 func getLogEventString(event notifier.LogEventType) string {

internal/httpd/internal_test.go

@@ -3799,13 +3799,13 @@ func TestEventsCSV(t *testing.T) {
 		Status: 1,
 	}
 	data := e.getCSVData()
-	assert.Equal(t, "OK", data[5])
+	assert.Equal(t, "OK", data[6])
 	e.Status = 2
 	data = e.getCSVData()
-	assert.Equal(t, "KO", data[5])
+	assert.Equal(t, "KO", data[6])
 	e.Status = 3
 	data = e.getCSVData()
-	assert.Equal(t, "Quota exceeded", data[5])
+	assert.Equal(t, "Quota exceeded", data[6])
 }
 
 func TestConfigsFromProvider(t *testing.T) {

internal/httpd/server.go

@@ -1429,7 +1429,7 @@ func (s *httpdServer) setupRESTAPIRoutes() {
 					updateUserQuotaUsage)
 				router.With(s.checkPerms(dataprovider.PermAdminChangeUsers)).Put(quotasBasePath+"/users/{username}/transfer-usage",
 					updateUserTransferQuotaUsage)
-				router.With(s.checkPerms(dataprovider.PermAdminChangeUsers)).Put(quotasBasePath+"/folders/{name}/usage",
+				router.With(s.checkPerms(dataprovider.PermAdminManageFolders)).Put(quotasBasePath+"/folders/{name}/usage",
 					updateFolderQuotaUsage)
 				router.With(s.checkPerms(dataprovider.PermAdminViewDefender)).Get(defenderHosts, getDefenderHosts)
 				router.With(s.checkPerms(dataprovider.PermAdminViewDefender)).Get(defenderHosts+"/{id}", getDefenderHostByID)

internal/logger/logger.go

@@ -200,7 +200,7 @@ func ErrorToConsole(format string, v ...any) {
 }
 
 // TransferLog logs uploads or downloads
-func TransferLog(operation, path string, elapsed int64, size int64, user, connectionID, protocol, localAddr,
+func TransferLog(operation, path, virtualPath string, elapsed int64, size int64, user, connectionID, protocol, localAddr,
 	remoteAddr, ftpMode string, err error,
 ) {
 	var ev *zerolog.Event
@@ -218,6 +218,7 @@ func TransferLog(operation, path string, elapsed int64, size int64, user, connec
 		Int64("size_bytes", size).
 		Str("username", user).
 		Str("file_path", path).
+		Str("virtual_path", virtualPath).
 		Str("connection_id", connectionID).
 		Str("protocol", protocol)
 	if ftpMode != "" {
@@ -227,8 +228,9 @@ func TransferLog(operation, path string, elapsed int64, size int64, user, connec
 }
 
 // CommandLog logs an SFTP/SCP/SSH command
-func CommandLog(command, path, target, user, fileMode, connectionID, protocol string, uid, gid int, atime, mtime,
-	sshCommand string, size int64, localAddr, remoteAddr string, elapsed int64) {
+func CommandLog(command, path, target, virtualPath, virtualTarget, user, fileMode, connectionID, protocol string,
+	uid, gid int, atime, mtime, sshCommand string, size int64, localAddr, remoteAddr string, elapsed int64,
+) {
 	logger.Info().
 		Timestamp().
 		Str("sender", command).
@@ -237,6 +239,8 @@ func CommandLog(command, path, target, user, fileMode, connectionID, protocol st
 		Str("username", user).
 		Str("file_path", path).
 		Str("target_path", target).
+		Str("virtual_path", virtualPath).
+		Str("virtual_target_path", virtualTarget).
 		Str("filemode", fileMode).
 		Int("uid", uid).
 		Int("gid", gid).

internal/sftpd/ssh_cmd.go

@@ -283,9 +283,9 @@ func (c *sshCommand) sendExitStatus(err error) {
 		common.ExecuteActionNotification(c.connection.BaseConnection, common.OperationSSHCmd, cmdPath, vCmdPath, //nolint:errcheck
 			targetPath, vTargetPath, c.command, 0, err, elapsed, nil)
 		if err == nil {
-			logger.CommandLog(sshCommandLogSender, cmdPath, targetPath, c.connection.User.Username, "", c.connection.ID,
-				common.ProtocolSSH, -1, -1, "", "", c.connection.command, -1, c.connection.GetLocalAddress(),
-				c.connection.GetRemoteAddress(), elapsed)
+			logger.CommandLog(sshCommandLogSender, cmdPath, targetPath, vCmdPath, vTargetPath,
+				c.connection.User.Username, "", c.connection.ID, common.ProtocolSSH, -1, -1, "", "",
+				c.connection.command, -1, c.connection.GetLocalAddress(), c.connection.GetRemoteAddress(), elapsed)
 		}
 	}
 }