Compare commits

...
Author SHA1 Message Date
lishuang 8b8265d77f feat: replace front. 2026-02-14 23:56:36 +08:00
seaheart ceecc95c29 upgrade version 2026-02-14 23:18:57 +08:00
lishuang 27c48446ce feat: add auto clear feature. ready to release 4.2.0 2026-02-14 23:04:59 +08:00
seaheart 1d65c0e8d6 Support chunk upload and breakpoint resume feature 2026-02-08 16:31:56 +08:00
lishuang 6b09c1ffac feat: find service back. 2025-09-22 22:37:40 +08:00
lishuang 07d8e7e0b1 feat: show error log. 2025-09-22 22:12:50 +08:00
lishuang e80e41266a feat: show error log. 2025-09-22 22:05:57 +08:00
lishuang 640d543487 feat: show error log. 2025-09-22 21:59:06 +08:00
zicla aef3cee03c chmod x 2025-09-22 21:48:23 +08:00
lishuang a9d843d885 feat: upgrade to 4.1.3 2025-09-22 21:46:25 +08:00
lishuang bcc9d19916 feat: upgrade to 4.1.3 2025-09-21 23:27:38 +08:00
seaheart 23d2d40613 fix: space dir download logic 2025-09-14 23:10:53 +08:00
lishuang f873a4aeb9 feat: upgrade to 4.1.2 2025-02-24 23:01:44 +08:00
eyeblue a49495268f Merge pull request #273 from eyebluecn/fix-batch-download
fix: space files from different space member download batch error
2025-02-24 22:58:15 +08:00
seaheart 389b1bdd2e fix: batch download 2025-02-23 17:14:55 +08:00
eyeblue 09c903ec2d Merge pull request #265 from eyebluecn/fix-space-matter-preview
fix: space matter preview token logic
2024-10-06 21:54:56 +08:00
43 changed files with 1593 additions and 80 deletions
+5 -2
View File
@@ -9,10 +9,13 @@
2. build/pack/build.sh
VERSION_NAME=tank-x.x.x
3. core/config.go
3. build/pack/cross-build.sh
VERSION_NAME=tank-x.x.x
4. core/config.go
VERSION = x.x.x
4. readme.md 修改 eyeblue/tank:x.x.x
5. readme.md 修改 eyeblue/tank:x.x.x
eyeblue/tank:x.x.x
+2 -2
View File
@@ -2,12 +2,12 @@
[English Version](https://tank-doc.eyeblue.cn/en)
# 蓝眼云盘(4.1.1)
# 蓝眼云盘(4.2.0)
[在线Demo](https://tanker.eyeblue.cn) (体验账号: demo 密码:123456)
蓝眼云盘是蓝眼开源系列代表作品之一,致力于打造精致,优雅,简约的云盘。核心功能如下:
1. 文件列表
2. 文件/文件夹上传 + 拖拽上传
2. 文件/文件夹上传 + 拖拽上传 + 分片/断点续传
3. 共享空间
3. 文件分享
4. 文件监控统计
+21 -19
View File
@@ -1,27 +1,29 @@
{
"files": {
"main.css": "/static/css/main.8f121d4f.css",
"main.js": "/static/js/main.e27b1761.js",
"main.css": "/static/css/main.3e2e293d.css",
"main.js": "/static/js/main.1a935cc6.js",
"static/js/833.2731ecbb.chunk.js": "/static/js/833.2731ecbb.chunk.js",
"static/media/logo.png": "/static/media/logo.847e54ef7fb4b744fad4.png",
"static/media/empty.svg": "/static/media/empty.087b383305fa7993cbf2fd9ccfd2cdaf.svg",
"static/media/psd.svg": "/static/media/psd.d0769f3468941f83b2cd1491e6ac7d69.svg",
"static/media/xls.svg": "/static/media/xls.7a882f471ff07b1015705abf4945d6a3.svg",
"static/media/ppt.svg": "/static/media/ppt.b987e0e201d3a6ebb0f83c5444521626.svg",
"static/media/doc.svg": "/static/media/doc.bc5d5331ff681915068a120f067a9dec.svg",
"static/media/pdf.svg": "/static/media/pdf.c92b8a844f37db16d4a72bf62a73138c.svg",
"static/media/video.svg": "/static/media/video.a1b07956cad84a3efc508d024286b30f.svg",
"static/media/file.svg": "/static/media/file.9f6f3fda1dd60a03fe595532e845852c.svg",
"static/media/text.svg": "/static/media/text.8aeb382b647dad745f0e9c2f11c349d2.svg",
"static/media/audio.svg": "/static/media/audio.0d111f7f828e8ea78c83b47d23e771a2.svg",
"static/media/image.svg": "/static/media/image.333c8c7c7df48b383c852de5ce99c33c.svg",
"static/media/folder.svg": "/static/media/folder.fce5913565dc18a92c7bbbe19ba6c4d0.svg",
"static/media/archive.svg": "/static/media/archive.982c48675c38139800a2f45d7c86a661.svg",
"static/media/empty.svg": "/static/media/empty.1ac6f6fb3e1d8c985f32df632ded4fe1.svg",
"static/media/psd.svg": "/static/media/psd.f2767f53369b5e75d7708a857d8381f4.svg",
"static/media/xls.svg": "/static/media/xls.9f55e13113424a3cc3316acbf072f072.svg",
"static/media/ppt.svg": "/static/media/ppt.8696321d8f3893a15681473769c8283c.svg",
"static/media/doc.svg": "/static/media/doc.017d14ee10ef3d2427494d1b0e52b03e.svg",
"static/media/pdf.svg": "/static/media/pdf.67d0061b2cef7912ed36b895cb8a328a.svg",
"static/media/video.svg": "/static/media/video.09bf3324c0e699b3611bf6bb0fc1b50e.svg",
"static/media/file.svg": "/static/media/file.b6be968e09dc04a45ec49a0dbcef53da.svg",
"static/media/text.svg": "/static/media/text.fa66aadee4b7dd9d51f4d1c863e5f75e.svg",
"static/media/audio.svg": "/static/media/audio.59dab6a9512494e5a34c8a424ab68f39.svg",
"static/media/image.svg": "/static/media/image.ee35022875b37a7962afa08bce8bc1fd.svg",
"static/media/folder.svg": "/static/media/folder.b4ce8a2960e8da846081c692d3065dff.svg",
"static/media/archive.svg": "/static/media/archive.f58837716d1fc6f2e61dc1618dad27a6.svg",
"index.html": "/index.html",
"main.8f121d4f.css.map": "/static/css/main.8f121d4f.css.map",
"main.e27b1761.js.map": "/static/js/main.e27b1761.js.map"
"main.3e2e293d.css.map": "/static/css/main.3e2e293d.css.map",
"main.1a935cc6.js.map": "/static/js/main.1a935cc6.js.map",
"833.2731ecbb.chunk.js.map": "/static/js/833.2731ecbb.chunk.js.map"
},
"entrypoints": [
"static/css/main.8f121d4f.css",
"static/js/main.e27b1761.js"
"static/css/main.3e2e293d.css",
"static/js/main.1a935cc6.js"
]
}
+1 -1
View File
@@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="eyeblue tank"/><title>EyeblueTank</title><script defer="defer" src="/static/js/main.e27b1761.js"></script><link href="/static/css/main.8f121d4f.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run EyeblueTank.</noscript><div id="root"></div></body></html>
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="/favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="eyeblue tank"/><title>EyeblueTank</title><script defer="defer" src="/static/js/main.1a935cc6.js"></script><link href="/static/css/main.3e2e293d.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run EyeblueTank.</noscript><div id="root"></div></body></html>
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 855 B

After

Width:  |  Height:  |  Size: 855 B

Before

Width:  |  Height:  |  Size: 1.3 KiB

After

Width:  |  Height:  |  Size: 1.3 KiB

Before

Width:  |  Height:  |  Size: 2.1 KiB

After

Width:  |  Height:  |  Size: 2.1 KiB

Before

Width:  |  Height:  |  Size: 5.7 KiB

After

Width:  |  Height:  |  Size: 5.7 KiB

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

Before

Width:  |  Height:  |  Size: 949 B

After

Width:  |  Height:  |  Size: 949 B

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

Before

Width:  |  Height:  |  Size: 2.1 KiB

After

Width:  |  Height:  |  Size: 2.1 KiB

Before

Width:  |  Height:  |  Size: 2.5 KiB

After

Width:  |  Height:  |  Size: 2.5 KiB

Before

Width:  |  Height:  |  Size: 3.2 KiB

After

Width:  |  Height:  |  Size: 3.2 KiB

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

Before

Width:  |  Height:  |  Size: 1.8 KiB

After

Width:  |  Height:  |  Size: 1.8 KiB

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

+1 -1
View File
@@ -11,7 +11,7 @@
@REM prepare the variables.
@REM version name
SET VERSION_NAME=tank-4.1.1
SET VERSION_NAME=tank-4.2.0
ECHO VERSION_NAME: %VERSION_NAME%
@REM golang proxy
SET GOPROXY=https://goproxy.cn
+1 -1
View File
@@ -10,7 +10,7 @@ set -e
#prepare the variables.
# version name
VERSION_NAME=tank-4.1.1
VERSION_NAME=tank-4.2.0
echo "VERSION_NAME: ${VERSION_NAME}"
# golang proxy
GOPROXY=https://goproxy.cn
+379
View File
@@ -0,0 +1,379 @@
#!/bin/bash
#set -e
###########################################################################
#
# Tank cross-compilation build script
# Supports building for multiple platforms and architectures
#
###########################################################################
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Print usage information
usage() {
echo "Usage: $0 [OPTIONS]"
echo ""
echo "Options:"
echo " -v, --version VERSION Version name (default: tank-4.2.0)"
echo " -o, --os OS Target OS: linux, windows, darwin, or 'all'"
echo " -a, --arch ARCH Target architecture: amd64, arm64, 386, or 'all'"
echo " -p, --platform PLATFORM Specific platform (OS/ARCH format, e.g. linux/amd64)"
echo " --list-platforms List all supported platforms"
echo " --clean Clean dist directory before building"
echo " -h, --help Show this help message"
echo ""
echo "Examples:"
echo " $0 # Build for current platform"
echo " $0 -o linux -a amd64 # Build for Linux AMD64"
echo " $0 -p linux/arm64 # Build for Linux ARM64"
echo " $0 -o all -a amd64 # Build for all OS with AMD64"
echo " $0 -o all -a all # Build for all supported platforms"
echo " $0 --clean -o all -a all # Clean and build all platforms"
exit 1
}
# List supported platforms
list_platforms() {
echo "Supported platforms:"
echo " linux/amd64 - Linux 64-bit"
echo " linux/arm64 - Linux ARM64"
echo " linux/386 - Linux 32-bit"
echo " windows/amd64 - Windows 64-bit"
echo " windows/arm64 - Windows ARM64"
echo " windows/386 - Windows 32-bit"
echo " darwin/amd64 - macOS Intel"
echo " darwin/arm64 - macOS Apple Silicon"
exit 0
}
# Log functions
log_info() {
echo -e "${BLUE}[INFO]${NC} $1"
}
log_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
log_warn() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
# Default values
VERSION_NAME="tank-4.2.0"
TARGET_OS=""
TARGET_ARCH=""
SPECIFIC_PLATFORM=""
BUILD_ALL_OS=false
BUILD_ALL_ARCH=false
CLEAN_DIST=false
# Parse command line arguments
while [[ $# -gt 0 ]]; do
case $1 in
-v|--version)
VERSION_NAME="$2"
shift 2
;;
-o|--os)
if [[ "$2" == "all" ]]; then
BUILD_ALL_OS=true
else
TARGET_OS="$2"
fi
shift 2
;;
-a|--arch)
if [[ "$2" == "all" ]]; then
BUILD_ALL_ARCH=true
else
TARGET_ARCH="$2"
fi
shift 2
;;
-p|--platform)
SPECIFIC_PLATFORM="$2"
shift 2
;;
--list-platforms)
list_platforms
;;
--clean)
CLEAN_DIST=true
shift
;;
-h|--help)
usage
;;
*)
log_error "Unknown option: $1"
usage
;;
esac
done
# Parse specific platform
if [[ -n "$SPECIFIC_PLATFORM" ]]; then
if [[ "$SPECIFIC_PLATFORM" =~ ^([^/]+)/([^/]+)$ ]]; then
TARGET_OS="${BASH_REMATCH[1]}"
TARGET_ARCH="${BASH_REMATCH[2]}"
else
log_error "Invalid platform format. Use OS/ARCH format (e.g., linux/amd64)"
exit 1
fi
fi
# Set defaults if nothing specified
if [[ -z "$TARGET_OS" && "$BUILD_ALL_OS" == false ]]; then
TARGET_OS=$(go env GOOS)
fi
if [[ -z "$TARGET_ARCH" && "$BUILD_ALL_ARCH" == false ]]; then
TARGET_ARCH=$(go env GOARCH)
fi
# Define supported platforms as a simple list
SUPPORTED_PLATFORMS="linux/amd64 linux/arm64 linux/386 windows/amd64 windows/arm64 windows/386 darwin/amd64 darwin/arm64"
# Check if platform is supported
is_platform_supported() {
local platform="$1"
echo "$SUPPORTED_PLATFORMS" | grep -q "\b$platform\b"
}
# Validate platform
validate_platform() {
local os=$1
local arch=$2
local platform="${os}/${arch}"
if ! is_platform_supported "$platform"; then
log_warn "Platform ${platform} may not be fully tested"
return 1
fi
log_info "Platform ${platform} supported"
return 0
}
# Setup directories
GOPROXY=https://goproxy.cn
PACK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
BUILD_DIR=$(dirname ${PACK_DIR})
PROJECT_DIR=$(dirname ${BUILD_DIR})
DIST_DIR=${PROJECT_DIR}/tmp/dist
log_info "=== Tank Cross-Compilation Build ==="
log_info "Version: ${VERSION_NAME}"
log_info "GOPROXY: ${GOPROXY}"
log_info "Project: ${PROJECT_DIR}"
log_info "Dist: ${DIST_DIR}"
# Clean dist directory if requested
if [[ "$CLEAN_DIST" == true ]]; then
log_info "Cleaning dist directory..."
rm -rf ${DIST_DIR}
fi
# Create dist directory
mkdir -p ${DIST_DIR}
# Build function for specific platform
build_platform() {
local os=$1
local arch=$2
local platform="${os}/${arch}"
log_info "Building ${platform}..."
# Validate platform
validate_platform "$os" "$arch"
# Determine binary name and archive format
local binary_name="tank"
local archive_ext="tar.gz"
if [[ "$os" == "windows" ]]; then
binary_name="tank.exe"
archive_ext="zip"
fi
# Setup paths
local file_name="${VERSION_NAME}.${os}-${arch}.${archive_ext}"
local component_dir="${DIST_DIR}/${VERSION_NAME}-${os}-${arch}"
local dist_path="${DIST_DIR}/${file_name}"
log_info " Target: ${platform}"
log_info " Binary: ${binary_name}"
log_info " Output: ${file_name}"
# Clean previous build
if [[ -d "$component_dir" ]]; then
rm -rf "$component_dir"
fi
mkdir -p "$component_dir"
# Change to project directory
cd ${PROJECT_DIR}
# Cross-compile
log_info " Cross-compiling..."
export GOOS="$os"
export GOARCH="$arch"
export GOPROXY="$GOPROXY"
export CGO_ENABLED=0 # Disable CGO for cross-compilation
log_info " Starting go build..." # 添加此行
if ! go build -mod=readonly -ldflags="-s -w" -o "${binary_name}"; then
log_error "Failed to build for ${platform}"
return 1
fi
log_info " go build completed" # 添加此行
# Copy binary
log_info " Copying binary..."
cp "./${binary_name}" "${component_dir}/"
# Copy configuration and static files
log_info " Copying build files..."
if [[ -d "${BUILD_DIR}/conf" ]]; then
cp -r "${BUILD_DIR}/conf" "${component_dir}/"
fi
if [[ -d "${BUILD_DIR}/html" ]]; then
cp -r "${BUILD_DIR}/html" "${component_dir}/"
fi
if [[ -d "${BUILD_DIR}/service" ]]; then
cp -r "${BUILD_DIR}/service" "${component_dir}/"
fi
# Create archive
log_info " Creating archive..."
cd ${DIST_DIR}
if [[ "$archive_ext" == "zip" ]]; then
# Use zip for Windows
if command -v zip >/dev/null 2>&1; then
zip -r "${file_name}" "$(basename "$component_dir")" >/dev/null 2>&1
else
log_warn "zip command not found, using tar.gz instead"
tar -zcf "${VERSION_NAME}.${os}-${arch}.tar.gz" "$(basename "$component_dir")" >/dev/null 2>&1
fi
else
# Use tar.gz for Unix-like systems
tar -zcf "${file_name}" "$(basename "$component_dir")" >/dev/null 2>&1
fi
if [[ $? -ne 0 ]]; then
log_error "Failed to create archive for ${platform}"
return 1
fi
# Cleanup
cd ${PROJECT_DIR}
rm -f "./${binary_name}"
rm -rf "$component_dir"
# Get file size
local file_size=$(ls -lh "${DIST_DIR}/${file_name}" | awk '{print $5}')
log_success "Built ${file_name} (${file_size})"
return 0
}
# Build execution
build_count=0
success_count=0
failed_builds=()
if [[ "$BUILD_ALL_OS" == true && "$BUILD_ALL_ARCH" == true ]]; then
log_info "Building for all supported platforms..."
# Build all supported platforms explicitly
for os in linux windows darwin; do
for arch in amd64 arm64 386; do
platform="${os}/${arch}"
if is_platform_supported "$platform"; then
((build_count++))
if build_platform "$os" "$arch"; then
((success_count++))
else
failed_builds+=("$platform")
fi
fi
done
done
elif [[ "$BUILD_ALL_OS" == true ]]; then
log_info "Building for all OS with architecture: ${TARGET_ARCH}"
for os in linux windows darwin; do
platform="${os}/${TARGET_ARCH}"
if is_platform_supported "$platform"; then
((build_count++))
if build_platform "$os" "$TARGET_ARCH"; then
((success_count++))
else
failed_builds+=("$platform")
fi
fi
done
elif [[ "$BUILD_ALL_ARCH" == true ]]; then
log_info "Building for OS ${TARGET_OS} with all architectures"
for arch in amd64 arm64 386; do
platform="${TARGET_OS}/${arch}"
if is_platform_supported "$platform"; then
((build_count++))
if build_platform "$TARGET_OS" "$arch"; then
((success_count++))
else
failed_builds+=("$platform")
fi
fi
done
else
log_info "Building for ${TARGET_OS}/${TARGET_ARCH}"
((build_count++))
if build_platform "$TARGET_OS" "$TARGET_ARCH"; then
((success_count++))
else
failed_builds+=("${TARGET_OS}/${TARGET_ARCH}")
fi
fi
# Build summary
echo ""
log_info "=== Build Summary ==="
echo "Total builds: ${build_count}"
echo "Successful: ${success_count}"
echo "Failed: $((build_count - success_count))"
if [[ ${#failed_builds[@]} -gt 0 ]]; then
echo ""
log_warn "Failed builds:"
for platform in "${failed_builds[@]}"; do
echo " - $platform"
done
fi
if [[ $success_count -gt 0 ]]; then
echo ""
log_success "Generated files:"
ls -lh ${DIST_DIR}/*.{tar.gz,zip} 2>/dev/null | awk '{print " " $9 " (" $5 ")"}'
echo ""
log_info "Files located in: ${DIST_DIR}"
fi
echo ""
if [[ $((build_count - success_count)) -eq 0 ]]; then
log_success "All builds completed successfully!"
exit 0
else
log_error "Some builds failed!"
exit 1
fi
+3 -3
View File
@@ -11,10 +11,10 @@ const (
DEFAULT_SERVER_PORT = 6010
//db table's prefix. tank41_ means current version is tank:4.1.x
TABLE_PREFIX = "tank41_"
//db table's prefix. tank42_ means current version is tank:4.2.x
TABLE_PREFIX = "tank42_"
VERSION = "4.1.1"
VERSION = "4.2.0"
)
type Config interface {
+161
View File
@@ -0,0 +1,161 @@
package rest
import (
"github.com/eyebluecn/tank/code/core"
"github.com/eyebluecn/tank/code/tool/result"
"github.com/eyebluecn/tank/code/tool/util"
"net/http"
)
// ChunkController handles chunked upload API endpoints
type ChunkController struct {
BaseController
chunkService *ChunkService
matterDao *MatterDao
uploadSessionDao *UploadSessionDao
uploadChunkDao *UploadChunkDao
}
func (this *ChunkController) Init() {
this.BaseController.Init()
b := core.CONTEXT.GetBean(this.chunkService)
if b, ok := b.(*ChunkService); ok {
this.chunkService = b
}
b = core.CONTEXT.GetBean(this.matterDao)
if b, ok := b.(*MatterDao); ok {
this.matterDao = b
}
b = core.CONTEXT.GetBean(this.uploadSessionDao)
if b, ok := b.(*UploadSessionDao); ok {
this.uploadSessionDao = b
}
b = core.CONTEXT.GetBean(this.uploadChunkDao)
if b, ok := b.(*UploadChunkDao); ok {
this.uploadChunkDao = b
}
}
func (this *ChunkController) RegisterRoutes() map[string]func(writer http.ResponseWriter, request *http.Request) {
routeMap := make(map[string]func(writer http.ResponseWriter, request *http.Request))
// Chunked upload endpoints
routeMap["/api/chunk/create/session"] = this.Wrap(this.CreateSession, USER_ROLE_USER)
routeMap["/api/chunk/upload"] = this.Wrap(this.Upload, USER_ROLE_USER)
routeMap["/api/chunk/merge"] = this.Wrap(this.Merge, USER_ROLE_USER)
routeMap["/api/chunk/session/info"] = this.Wrap(this.SessionInfo, USER_ROLE_USER)
routeMap["/api/chunk/cancel"] = this.Wrap(this.Cancel, USER_ROLE_USER)
routeMap["/api/chunk/clean/expired"] = this.Wrap(this.CleanExpired, USER_ROLE_ADMINISTRATOR)
return routeMap
}
// CreateSession creates a new chunked upload session
// Parameters:
// - puuid: parent directory uuid
// - filename: target filename
// - totalSize: total file size in bytes
// - chunkSize: size of each chunk (optional, default 5MB)
// - privacy: whether the file is private (optional, default true)
// - fileMd5: MD5 of the whole file (optional, for verification)
// - spaceUuid: space uuid (optional, default user's private space)
func (this *ChunkController) CreateSession(writer http.ResponseWriter, request *http.Request) *result.WebResult {
puuid := util.ExtractRequestString(request, "puuid")
filename := util.ExtractRequestString(request, "filename")
totalSize := util.ExtractRequestInt64(request, "totalSize")
chunkSize := util.ExtractRequestOptionalInt64(request, "chunkSize", DEFAULT_CHUNK_SIZE)
privacy := util.ExtractRequestOptionalBool(request, "privacy", true)
fileMd5 := util.ExtractRequestOptionalString(request, "fileMd5", "")
user := this.checkUser(request)
spaceUuid := util.ExtractRequestOptionalString(request, "spaceUuid", user.SpaceUuid)
space := this.spaceService.CheckWritableByUuid(request, user, spaceUuid)
dirMatter := this.matterDao.CheckWithRootByUuid(puuid, space)
session := this.chunkService.CreateSession(request, user, space, dirMatter, filename, totalSize, chunkSize, privacy, fileMd5)
return this.Success(session)
}
// Upload uploads a single chunk
// Parameters:
// - sessionUuid: upload session uuid
// - chunkIndex: chunk index (0-based)
// - file: chunk file data
// - chunkMd5: MD5 of the chunk (optional, for verification)
// - spaceUuid: space uuid (optional)
func (this *ChunkController) Upload(writer http.ResponseWriter, request *http.Request) *result.WebResult {
sessionUuid := util.ExtractRequestString(request, "sessionUuid")
chunkIndex := util.ExtractRequestInt(request, "chunkIndex")
chunkMd5 := util.ExtractRequestOptionalString(request, "chunkMd5", "")
user := this.checkUser(request)
spaceUuid := util.ExtractRequestOptionalString(request, "spaceUuid", user.SpaceUuid)
space := this.spaceService.CheckWritableByUuid(request, user, spaceUuid)
file, _, err := request.FormFile("file")
if err != nil {
panic(result.BadRequest("file is required: %s", err.Error()))
}
defer file.Close()
chunk := this.chunkService.UploadChunk(request, sessionUuid, chunkIndex, file, chunkMd5, user, space)
return this.Success(chunk)
}
// Merge merges all chunks into the final file
// Parameters:
// - sessionUuid: upload session uuid
// - spaceUuid: space uuid (optional)
func (this *ChunkController) Merge(writer http.ResponseWriter, request *http.Request) *result.WebResult {
sessionUuid := util.ExtractRequestString(request, "sessionUuid")
user := this.checkUser(request)
spaceUuid := util.ExtractRequestOptionalString(request, "spaceUuid", user.SpaceUuid)
space := this.spaceService.CheckWritableByUuid(request, user, spaceUuid)
matter := this.chunkService.MergeChunks(request, sessionUuid, user, space)
return this.Success(matter)
}
// SessionInfo gets the upload session info including uploaded chunks
// Parameters:
// - sessionUuid: upload session uuid
func (this *ChunkController) SessionInfo(writer http.ResponseWriter, request *http.Request) *result.WebResult {
sessionUuid := util.ExtractRequestString(request, "sessionUuid")
user := this.checkUser(request)
sessionInfo := this.chunkService.GetSessionInfo(sessionUuid, user)
return this.Success(sessionInfo)
}
// Cancel cancels an upload session
// Parameters:
// - sessionUuid: upload session uuid
// - spaceUuid: space uuid (optional)
func (this *ChunkController) Cancel(writer http.ResponseWriter, request *http.Request) *result.WebResult {
sessionUuid := util.ExtractRequestString(request, "sessionUuid")
user := this.checkUser(request)
spaceUuid := util.ExtractRequestOptionalString(request, "spaceUuid", user.SpaceUuid)
space := this.spaceService.CheckWritableByUuid(request, user, spaceUuid)
this.chunkService.CancelSession(sessionUuid, user, space)
return this.Success("OK")
}
// CleanExpired cleans up expired upload sessions (admin only)
func (this *ChunkController) CleanExpired(writer http.ResponseWriter, request *http.Request) *result.WebResult {
this.chunkService.CleanExpiredSessions()
return this.Success("OK")
}
+212
View File
@@ -0,0 +1,212 @@
package rest
import (
"github.com/eyebluecn/tank/code/core"
"github.com/eyebluecn/tank/code/tool/result"
"github.com/eyebluecn/tank/code/tool/uuid"
"time"
)
// UploadSessionDao handles database operations for UploadSession
type UploadSessionDao struct {
BaseDao
}
// FindByUuid finds an upload session by uuid
func (this *UploadSessionDao) FindByUuid(sessionUuid string) *UploadSession {
var entity = &UploadSession{}
db := core.CONTEXT.GetDB().Where("uuid = ?", sessionUuid).First(entity)
if db.Error != nil {
if db.Error.Error() == result.DB_ERROR_NOT_FOUND {
return nil
} else {
panic(db.Error)
}
}
return entity
}
// CheckByUuid checks if session exists, panics if not found
func (this *UploadSessionDao) CheckByUuid(sessionUuid string) *UploadSession {
entity := this.FindByUuid(sessionUuid)
if entity == nil {
panic(result.NotFound("upload session not found with uuid = %s", sessionUuid))
}
return entity
}
// FindByUserUuidAndFileMd5 finds session by user and file MD5 (for quick upload)
func (this *UploadSessionDao) FindByUserUuidAndFileMd5(userUuid string, fileMd5 string) *UploadSession {
var entity = &UploadSession{}
db := core.CONTEXT.GetDB().Where("user_uuid = ? AND file_md5 = ? AND status = ?",
userUuid, fileMd5, UPLOAD_SESSION_STATUS_COMPLETED).First(entity)
if db.Error != nil {
if db.Error.Error() == result.DB_ERROR_NOT_FOUND {
return nil
} else {
panic(db.Error)
}
}
return entity
}
// FindBySpaceUuidAndFileMd5 finds completed session by space and file MD5
func (this *UploadSessionDao) FindBySpaceUuidAndFileMd5(spaceUuid string, fileMd5 string) *UploadSession {
var entity = &UploadSession{}
db := core.CONTEXT.GetDB().Where("space_uuid = ? AND file_md5 = ? AND status = ?",
spaceUuid, fileMd5, UPLOAD_SESSION_STATUS_COMPLETED).First(entity)
if db.Error != nil {
if db.Error.Error() == result.DB_ERROR_NOT_FOUND {
return nil
} else {
panic(db.Error)
}
}
return entity
}
// Create creates a new upload session
func (this *UploadSessionDao) Create(session *UploadSession) *UploadSession {
timeUUID, _ := uuid.NewV4()
session.Uuid = timeUUID.String()
session.CreateTime = time.Now()
session.UpdateTime = time.Now()
session.Sort = time.Now().UnixNano() / 1e6
db := core.CONTEXT.GetDB().Create(session)
this.PanicError(db.Error)
return session
}
// Save updates an upload session
func (this *UploadSessionDao) Save(session *UploadSession) *UploadSession {
session.UpdateTime = time.Now()
db := core.CONTEXT.GetDB().Save(session)
this.PanicError(db.Error)
return session
}
// Delete deletes an upload session
func (this *UploadSessionDao) Delete(session *UploadSession) {
db := core.CONTEXT.GetDB().Delete(session)
this.PanicError(db.Error)
}
// DeleteByUuid deletes session by uuid
func (this *UploadSessionDao) DeleteByUuid(sessionUuid string) {
db := core.CONTEXT.GetDB().Where("uuid = ?", sessionUuid).Delete(UploadSession{})
this.PanicError(db.Error)
}
// FindExpiredSessions finds all expired sessions
func (this *UploadSessionDao) FindExpiredSessions() []*UploadSession {
var sessions []*UploadSession
db := core.CONTEXT.GetDB().Where("expire_time < ? AND status = ?",
time.Now(), UPLOAD_SESSION_STATUS_UPLOADING).Find(&sessions)
this.PanicError(db.Error)
return sessions
}
// FindByUserUuid finds all sessions for a user
func (this *UploadSessionDao) FindByUserUuid(userUuid string) []*UploadSession {
var sessions []*UploadSession
db := core.CONTEXT.GetDB().Where("user_uuid = ?", userUuid).Find(&sessions)
this.PanicError(db.Error)
return sessions
}
// DeleteByUserUuid deletes all sessions for a user
func (this *UploadSessionDao) DeleteByUserUuid(userUuid string) {
db := core.CONTEXT.GetDB().Where("user_uuid = ?", userUuid).Delete(UploadSession{})
this.PanicError(db.Error)
}
// UploadChunkDao handles database operations for UploadChunk
type UploadChunkDao struct {
BaseDao
}
// FindByUuid finds a chunk by uuid
func (this *UploadChunkDao) FindByUuid(chunkUuid string) *UploadChunk {
var entity = &UploadChunk{}
db := core.CONTEXT.GetDB().Where("uuid = ?", chunkUuid).First(entity)
if db.Error != nil {
if db.Error.Error() == result.DB_ERROR_NOT_FOUND {
return nil
} else {
panic(db.Error)
}
}
return entity
}
// FindBySessionUuidAndChunkIndex finds a chunk by session and index
func (this *UploadChunkDao) FindBySessionUuidAndChunkIndex(sessionUuid string, chunkIndex int) *UploadChunk {
var entity = &UploadChunk{}
db := core.CONTEXT.GetDB().Where("session_uuid = ? AND chunk_index = ?", sessionUuid, chunkIndex).First(entity)
if db.Error != nil {
if db.Error.Error() == result.DB_ERROR_NOT_FOUND {
return nil
} else {
panic(db.Error)
}
}
return entity
}
// FindBySessionUuid finds all chunks for a session
func (this *UploadChunkDao) FindBySessionUuid(sessionUuid string) []*UploadChunk {
var chunks []*UploadChunk
db := core.CONTEXT.GetDB().Where("session_uuid = ?", sessionUuid).Order("chunk_index asc").Find(&chunks)
this.PanicError(db.Error)
return chunks
}
// FindUploadedBySessionUuid finds all uploaded chunks for a session
func (this *UploadChunkDao) FindUploadedBySessionUuid(sessionUuid string) []*UploadChunk {
var chunks []*UploadChunk
db := core.CONTEXT.GetDB().Where("session_uuid = ? AND uploaded = ?", sessionUuid, true).Order("chunk_index asc").Find(&chunks)
this.PanicError(db.Error)
return chunks
}
// CountUploadedBySessionUuid counts uploaded chunks for a session
func (this *UploadChunkDao) CountUploadedBySessionUuid(sessionUuid string) int64 {
var count int64
db := core.CONTEXT.GetDB().Model(&UploadChunk{}).Where("session_uuid = ? AND uploaded = ?", sessionUuid, true).Count(&count)
this.PanicError(db.Error)
return count
}
// Create creates a new chunk record
func (this *UploadChunkDao) Create(chunk *UploadChunk) *UploadChunk {
timeUUID, _ := uuid.NewV4()
chunk.Uuid = timeUUID.String()
chunk.CreateTime = time.Now()
chunk.UpdateTime = time.Now()
chunk.Sort = time.Now().UnixNano() / 1e6
db := core.CONTEXT.GetDB().Create(chunk)
this.PanicError(db.Error)
return chunk
}
// Save updates a chunk record
func (this *UploadChunkDao) Save(chunk *UploadChunk) *UploadChunk {
chunk.UpdateTime = time.Now()
db := core.CONTEXT.GetDB().Save(chunk)
this.PanicError(db.Error)
return chunk
}
// Delete deletes a chunk record
func (this *UploadChunkDao) Delete(chunk *UploadChunk) {
db := core.CONTEXT.GetDB().Delete(chunk)
this.PanicError(db.Error)
}
// DeleteBySessionUuid deletes all chunks for a session
func (this *UploadChunkDao) DeleteBySessionUuid(sessionUuid string) {
db := core.CONTEXT.GetDB().Where("session_uuid = ?", sessionUuid).Delete(UploadChunk{})
this.PanicError(db.Error)
}
+76
View File
@@ -0,0 +1,76 @@
package rest
import (
"time"
)
const (
// Upload session status
UPLOAD_SESSION_STATUS_UPLOADING = "UPLOADING"
UPLOAD_SESSION_STATUS_MERGING = "MERGING"
UPLOAD_SESSION_STATUS_COMPLETED = "COMPLETED"
UPLOAD_SESSION_STATUS_ERROR = "ERROR"
// Chunk directory name
CHUNK_DIR = "chunk"
// Default chunk size: 50MB
DEFAULT_CHUNK_SIZE = 50 * 1024 * 1024
// Max chunk size: 100MB
MAX_CHUNK_SIZE = 100 * 1024 * 1024
// Min chunk size: 5MB (aligned with AWS S3 standard)
MIN_CHUNK_SIZE = 5 * 1024 * 1024
)
// UploadSession represents a chunked upload session
type UploadSession struct {
Uuid string `json:"uuid" gorm:"type:char(36);primary_key;unique"`
Sort int64 `json:"sort" gorm:"type:bigint(20) not null"`
UpdateTime time.Time `json:"updateTime" gorm:"type:timestamp not null;default:CURRENT_TIMESTAMP"`
CreateTime time.Time `json:"createTime" gorm:"type:timestamp not null;default:'2018-01-01 00:00:00'"`
UserUuid string `json:"userUuid" gorm:"type:char(36) not null;index:idx_upload_session_uu"`
SpaceUuid string `json:"spaceUuid" gorm:"type:char(36) not null;index:idx_upload_session_su"`
MatterUuid string `json:"matterUuid" gorm:"type:char(36)"` // Final matter uuid after merge
Puuid string `json:"puuid" gorm:"type:char(36) not null"` // Parent directory uuid
Filename string `json:"filename" gorm:"type:varchar(255) not null"` // Target filename
TotalSize int64 `json:"totalSize" gorm:"type:bigint(20) not null;default:0"` // Total file size
ChunkSize int64 `json:"chunkSize" gorm:"type:bigint(20) not null;default:52428800"` // Size of each chunk (default 50MB)
TotalChunks int `json:"totalChunks" gorm:"type:int not null;default:0"` // Total number of chunks
Privacy bool `json:"privacy" gorm:"type:tinyint(1) not null;default:1"` // Privacy flag
FileMd5 string `json:"fileMd5" gorm:"type:varchar(45)"` // MD5 of the whole file
Status string `json:"status" gorm:"type:varchar(20) not null;default:'UPLOADING'"` // Session status
ExpireTime time.Time `json:"expireTime" gorm:"type:timestamp not null;default:'2018-01-01 00:00:00'"` // Expiration time
}
// UploadChunk represents a single chunk in an upload session
type UploadChunk struct {
Uuid string `json:"uuid" gorm:"type:char(36);primary_key;unique"`
Sort int64 `json:"sort" gorm:"type:bigint(20) not null"`
UpdateTime time.Time `json:"updateTime" gorm:"type:timestamp not null;default:CURRENT_TIMESTAMP"`
CreateTime time.Time `json:"createTime" gorm:"type:timestamp not null;default:'2018-01-01 00:00:00'"`
SessionUuid string `json:"sessionUuid" gorm:"type:char(36) not null;index:idx_upload_chunk_su"` // Reference to UploadSession
ChunkIndex int `json:"chunkIndex" gorm:"type:int not null"` // Chunk sequence number (0-based)
ChunkSize int64 `json:"chunkSize" gorm:"type:bigint(20) not null;default:0"` // Actual size of this chunk
ChunkMd5 string `json:"chunkMd5" gorm:"type:varchar(45)"` // MD5 of this chunk
ChunkFilename string `json:"chunkFilename" gorm:"type:varchar(255) not null"` // Temp filename for this chunk
Uploaded bool `json:"uploaded" gorm:"type:tinyint(1) not null;default:0"` // Whether this chunk is uploaded
}
// Get the chunk temp directory for a space
func GetSpaceChunkDir(spaceName string) string {
return GetUserSpaceRootDir(spaceName) + "/" + CHUNK_DIR
}
// Get the session chunk directory
func GetSessionChunkDir(spaceName string, sessionUuid string) string {
return GetSpaceChunkDir(spaceName) + "/" + sessionUuid
}
// UploadSessionInfo is the response structure for session queries
type UploadSessionInfo struct {
Session *UploadSession `json:"session"`
UploadedChunks []int `json:"uploadedChunks"` // List of uploaded chunk indices
MissingChunks []int `json:"missingChunks"` // List of missing chunk indices
}
+581
View File
@@ -0,0 +1,581 @@
package rest
import (
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"net/http"
"os"
"sort"
"sync"
"time"
"github.com/eyebluecn/tank/code/core"
"github.com/eyebluecn/tank/code/tool/i18n"
"github.com/eyebluecn/tank/code/tool/result"
"github.com/eyebluecn/tank/code/tool/util"
)
// ChunkService handles chunked upload operations
//@Service
type ChunkService struct {
BaseBean
uploadSessionDao *UploadSessionDao
uploadChunkDao *UploadChunkDao
matterDao *MatterDao
matterService *MatterService
spaceDao *SpaceDao
userService *UserService
// chunkLocker is used to prevent concurrent uploads of the same chunk
chunkLocker sync.Map
// mergeLocker is used to prevent concurrent merge of the same session
mergeLocker sync.Map
}
func (this *ChunkService) Init() {
this.BaseBean.Init()
b := core.CONTEXT.GetBean(this.uploadSessionDao)
if b, ok := b.(*UploadSessionDao); ok {
this.uploadSessionDao = b
}
b = core.CONTEXT.GetBean(this.uploadChunkDao)
if b, ok := b.(*UploadChunkDao); ok {
this.uploadChunkDao = b
}
b = core.CONTEXT.GetBean(this.matterDao)
if b, ok := b.(*MatterDao); ok {
this.matterDao = b
}
b = core.CONTEXT.GetBean(this.matterService)
if b, ok := b.(*MatterService); ok {
this.matterService = b
}
b = core.CONTEXT.GetBean(this.spaceDao)
if b, ok := b.(*SpaceDao); ok {
this.spaceDao = b
}
b = core.CONTEXT.GetBean(this.userService)
if b, ok := b.(*UserService); ok {
this.userService = b
}
}
// CreateSession creates a new upload session
func (this *ChunkService) CreateSession(
request *http.Request,
user *User,
space *Space,
dirMatter *Matter,
filename string,
totalSize int64,
chunkSize int64,
privacy bool,
fileMd5 string,
) *UploadSession {
if user == nil {
panic(result.BadRequest("user cannot be nil."))
}
if dirMatter == nil {
panic(result.BadRequest("dirMatter cannot be nil."))
}
if dirMatter.Deleted {
panic(result.BadRequest("Dir has been deleted. Cannot upload under it."))
}
// Validate filename
if len(filename) > MATTER_NAME_MAX_LENGTH {
panic(result.BadRequestI18n(request, i18n.MatterNameLengthExceedLimit, len(filename), MATTER_NAME_MAX_LENGTH))
}
CheckMatterName(request, filename)
// Validate chunk size
if chunkSize < MIN_CHUNK_SIZE {
chunkSize = MIN_CHUNK_SIZE
}
if chunkSize > MAX_CHUNK_SIZE {
chunkSize = MAX_CHUNK_SIZE
}
// Validate total size
if totalSize <= 0 {
panic(result.BadRequest("totalSize must be positive"))
}
// Check space size limit
if space.SizeLimit >= 0 {
if totalSize > space.SizeLimit {
panic(result.BadRequestI18n(request, i18n.MatterSizeExceedLimit, util.HumanFileSize(totalSize), util.HumanFileSize(space.SizeLimit)))
}
}
// Check total space limit
if space.TotalSizeLimit >= 0 {
if space.TotalSize+totalSize > space.TotalSizeLimit {
panic(result.BadRequestI18n(request, i18n.MatterSizeExceedTotalLimit, util.HumanFileSize(space.TotalSize), util.HumanFileSize(space.TotalSizeLimit)))
}
}
// Check if file already exists
dbMatter := this.matterDao.FindBySpaceUuidAndPuuidAndDirAndName(space.Uuid, dirMatter.Uuid, false, filename)
if dbMatter != nil {
if dbMatter.Deleted {
panic(result.BadRequestI18n(request, i18n.MatterRecycleBinExist, filename))
} else {
panic(result.BadRequestI18n(request, i18n.MatterExist, filename))
}
}
// Calculate total chunks
totalChunks := int((totalSize + chunkSize - 1) / chunkSize)
// Create session
session := &UploadSession{
UserUuid: user.Uuid,
SpaceUuid: space.Uuid,
Puuid: dirMatter.Uuid,
Filename: filename,
TotalSize: totalSize,
ChunkSize: chunkSize,
TotalChunks: totalChunks,
Privacy: privacy,
FileMd5: fileMd5,
Status: UPLOAD_SESSION_STATUS_UPLOADING,
ExpireTime: time.Now().Add(24 * time.Hour), // 24 hours expiration
}
session = this.uploadSessionDao.Create(session)
// Create chunk directory
chunkDir := GetSessionChunkDir(space.Name, session.Uuid)
util.MakeDirAll(chunkDir)
this.logger.Info("Created upload session %s for file %s, total chunks: %d", session.Uuid, filename, totalChunks)
return session
}
// UploadChunk uploads a single chunk with optional MD5 verification
func (this *ChunkService) UploadChunk(
request *http.Request,
sessionUuid string,
chunkIndex int,
file io.Reader,
clientChunkMd5 string,
user *User,
space *Space,
) *UploadChunk {
// Check session
session := this.uploadSessionDao.CheckByUuid(sessionUuid)
// Validate session ownership
if session.UserUuid != user.Uuid {
panic(result.BadRequest("session does not belong to you"))
}
if session.Status != UPLOAD_SESSION_STATUS_UPLOADING {
panic(result.BadRequest("session status is not uploading"))
}
// Check expiration
if time.Now().After(session.ExpireTime) {
panic(result.BadRequest("upload session has expired"))
}
// Validate chunk index
if chunkIndex < 0 || chunkIndex >= session.TotalChunks {
panic(result.BadRequest("invalid chunk index: %d, total chunks: %d", chunkIndex, session.TotalChunks))
}
// Acquire chunk-level lock to prevent concurrent upload of the same chunk
chunkLockKey := fmt.Sprintf("%s_%d", sessionUuid, chunkIndex)
if _, loaded := this.chunkLocker.LoadOrStore(chunkLockKey, true); loaded {
panic(result.BadRequest("chunk %d is being uploaded by another request, please retry later", chunkIndex))
}
defer this.chunkLocker.Delete(chunkLockKey)
// Check if chunk already exists
existingChunk := this.uploadChunkDao.FindBySessionUuidAndChunkIndex(sessionUuid, chunkIndex)
if existingChunk != nil && existingChunk.Uploaded {
this.logger.Info("Chunk %d already uploaded for session %s", chunkIndex, sessionUuid)
return existingChunk
}
// Prepare chunk file path
chunkFilename := fmt.Sprintf("chunk_%d", chunkIndex)
chunkDir := GetSessionChunkDir(space.Name, sessionUuid)
chunkFilePath := chunkDir + "/" + chunkFilename
// Write chunk to disk (use 0600 for security - only owner can read/write)
destFile, err := os.OpenFile(chunkFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
panic(result.BadRequest("failed to create chunk file: %s", err.Error()))
}
// Calculate MD5 while writing
hash := md5.New()
multiWriter := io.MultiWriter(destFile, hash)
chunkSize, err := io.Copy(multiWriter, file)
destFile.Close()
if err != nil {
os.Remove(chunkFilePath)
panic(result.BadRequest("failed to write chunk: %s", err.Error()))
}
// Validate chunk size to prevent malicious oversized uploads
isLastChunk := chunkIndex == session.TotalChunks-1
if isLastChunk {
// Last chunk: should be <= session.ChunkSize and match expected remainder
expectedLastChunkSize := session.TotalSize - int64(session.TotalChunks-1)*session.ChunkSize
if chunkSize != expectedLastChunkSize {
os.Remove(chunkFilePath)
panic(result.BadRequest("last chunk size mismatch: expected %d, got %d", expectedLastChunkSize, chunkSize))
}
} else {
// Non-last chunks: should be exactly session.ChunkSize
if chunkSize != session.ChunkSize {
os.Remove(chunkFilePath)
panic(result.BadRequest("chunk size mismatch: expected %d, got %d", session.ChunkSize, chunkSize))
}
}
chunkMd5 := hex.EncodeToString(hash.Sum(nil))
// Verify chunk MD5 if client provided one
if clientChunkMd5 != "" && clientChunkMd5 != chunkMd5 {
os.Remove(chunkFilePath)
this.logger.Error("Chunk MD5 verification failed for chunk %d: client=%s, server=%s", chunkIndex, clientChunkMd5, chunkMd5)
panic(result.BadRequest("chunk MD5 verification failed: data may be corrupted during upload"))
}
this.logger.Info("Uploaded chunk %d for session %s, size: %d, md5: %s", chunkIndex, sessionUuid, chunkSize, chunkMd5)
// Create or update chunk record
var chunk *UploadChunk
if existingChunk != nil {
existingChunk.ChunkSize = chunkSize
existingChunk.ChunkMd5 = chunkMd5
existingChunk.ChunkFilename = chunkFilename
existingChunk.Uploaded = true
chunk = this.uploadChunkDao.Save(existingChunk)
} else {
chunk = &UploadChunk{
SessionUuid: sessionUuid,
ChunkIndex: chunkIndex,
ChunkSize: chunkSize,
ChunkMd5: chunkMd5,
ChunkFilename: chunkFilename,
Uploaded: true,
}
chunk = this.uploadChunkDao.Create(chunk)
}
return chunk
}
// GetSessionInfo gets the upload session info including uploaded chunks
func (this *ChunkService) GetSessionInfo(sessionUuid string, user *User) *UploadSessionInfo {
session := this.uploadSessionDao.CheckByUuid(sessionUuid)
// Validate ownership
if session.UserUuid != user.Uuid {
panic(result.BadRequest("session does not belong to you"))
}
// Get uploaded chunks
uploadedChunks := this.uploadChunkDao.FindUploadedBySessionUuid(sessionUuid)
uploadedIndices := make([]int, 0)
uploadedMap := make(map[int]bool)
for _, chunk := range uploadedChunks {
uploadedIndices = append(uploadedIndices, chunk.ChunkIndex)
uploadedMap[chunk.ChunkIndex] = true
}
// Calculate missing chunks
missingIndices := make([]int, 0)
for i := 0; i < session.TotalChunks; i++ {
if !uploadedMap[i] {
missingIndices = append(missingIndices, i)
}
}
return &UploadSessionInfo{
Session: session,
UploadedChunks: uploadedIndices,
MissingChunks: missingIndices,
}
}
// MergeChunks merges all chunks into final file
func (this *ChunkService) MergeChunks(
request *http.Request,
sessionUuid string,
user *User,
space *Space,
) *Matter {
// Use session-level lock instead of user-level lock to allow concurrent uploads of different files
if _, loaded := this.mergeLocker.LoadOrStore(sessionUuid, true); loaded {
panic(result.BadRequest("merge operation is already in progress for this session"))
}
defer this.mergeLocker.Delete(sessionUuid)
session := this.uploadSessionDao.CheckByUuid(sessionUuid)
// Validate ownership
if session.UserUuid != user.Uuid {
panic(result.BadRequest("session does not belong to you"))
}
// Check session expiration
if time.Now().After(session.ExpireTime) {
panic(result.BadRequest("upload session has expired, please start a new upload"))
}
if session.Status == UPLOAD_SESSION_STATUS_COMPLETED {
// Already completed, return the matter
if session.MatterUuid != "" {
return this.matterDao.CheckByUuid(session.MatterUuid)
}
panic(result.BadRequest("session completed but matter not found"))
}
// Allow retry from ERROR status (e.g., previous merge failed)
if session.Status != UPLOAD_SESSION_STATUS_UPLOADING && session.Status != UPLOAD_SESSION_STATUS_ERROR {
panic(result.BadRequest("session status is not uploading or error, current: %s", session.Status))
}
// If retrying from ERROR status, reset to UPLOADING first
if session.Status == UPLOAD_SESSION_STATUS_ERROR {
this.logger.Info("Retrying merge for session %s from ERROR status", sessionUuid)
session.Status = UPLOAD_SESSION_STATUS_UPLOADING
this.uploadSessionDao.Save(session)
}
// Check if all chunks are uploaded
uploadedCount := this.uploadChunkDao.CountUploadedBySessionUuid(sessionUuid)
if int(uploadedCount) < session.TotalChunks {
panic(result.BadRequest("not all chunks uploaded: %d/%d", uploadedCount, session.TotalChunks))
}
// Update status to merging
session.Status = UPLOAD_SESSION_STATUS_MERGING
this.uploadSessionDao.Save(session)
// Get parent directory
var dirMatter *Matter
if session.Puuid == MATTER_ROOT {
dirMatter = NewRootMatter(space)
} else {
dirMatter = this.matterDao.CheckByUuid(session.Puuid)
}
// Prepare final file path
dirAbsolutePath := dirMatter.AbsolutePath()
util.MakeDirAll(dirAbsolutePath)
fileAbsolutePath := dirAbsolutePath + "/" + session.Filename
// Check if file exists
if util.PathExists(fileAbsolutePath) {
this.logger.Error("%s exists, removing it.", fileAbsolutePath)
os.Remove(fileAbsolutePath)
}
// Create final file (use 0644 for normal file permissions)
destFile, err := os.OpenFile(fileAbsolutePath, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
session.Status = UPLOAD_SESSION_STATUS_ERROR
this.uploadSessionDao.Save(session)
panic(result.BadRequest("failed to create destination file: %s", err.Error()))
}
// Get all chunks sorted by index
chunks := this.uploadChunkDao.FindBySessionUuid(sessionUuid)
sort.Slice(chunks, func(i, j int) bool {
return chunks[i].ChunkIndex < chunks[j].ChunkIndex
})
chunkDir := GetSessionChunkDir(space.Name, sessionUuid)
// Merge chunks
var totalWritten int64 = 0
for _, chunk := range chunks {
chunkPath := chunkDir + "/" + chunk.ChunkFilename
chunkFile, err := os.Open(chunkPath)
if err != nil {
destFile.Close()
os.Remove(fileAbsolutePath)
session.Status = UPLOAD_SESSION_STATUS_ERROR
this.uploadSessionDao.Save(session)
panic(result.BadRequest("failed to open chunk %d: %s", chunk.ChunkIndex, err.Error()))
}
written, err := io.Copy(destFile, chunkFile)
chunkFile.Close()
if err != nil {
destFile.Close()
os.Remove(fileAbsolutePath)
session.Status = UPLOAD_SESSION_STATUS_ERROR
this.uploadSessionDao.Save(session)
panic(result.BadRequest("failed to write chunk %d: %s", chunk.ChunkIndex, err.Error()))
}
totalWritten += written
}
destFile.Close()
this.logger.Info("Merged %d chunks into file %s, total size: %d", len(chunks), session.Filename, totalWritten)
// Verify file size
if totalWritten != session.TotalSize {
os.Remove(fileAbsolutePath)
session.Status = UPLOAD_SESSION_STATUS_ERROR
this.uploadSessionDao.Save(session)
panic(result.BadRequest("merged file size mismatch: expected %d, got %d", session.TotalSize, totalWritten))
}
// Verify file MD5 if provided
if session.FileMd5 != "" {
mergedFile, err := os.Open(fileAbsolutePath)
if err != nil {
os.Remove(fileAbsolutePath)
session.Status = UPLOAD_SESSION_STATUS_ERROR
this.uploadSessionDao.Save(session)
panic(result.BadRequest("failed to open merged file for MD5 verification: %s", err.Error()))
}
hash := md5.New()
_, err = io.Copy(hash, mergedFile)
mergedFile.Close()
if err != nil {
os.Remove(fileAbsolutePath)
session.Status = UPLOAD_SESSION_STATUS_ERROR
this.uploadSessionDao.Save(session)
panic(result.BadRequest("failed to calculate MD5 of merged file: %s", err.Error()))
}
calculatedMd5 := hex.EncodeToString(hash.Sum(nil))
if calculatedMd5 != session.FileMd5 {
os.Remove(fileAbsolutePath)
session.Status = UPLOAD_SESSION_STATUS_ERROR
this.uploadSessionDao.Save(session)
this.logger.Error("MD5 verification failed for session %s: expected %s, got %s", sessionUuid, session.FileMd5, calculatedMd5)
panic(result.BadRequest("MD5 verification failed: file may be corrupted during upload"))
}
this.logger.Info("MD5 verification passed for session %s", sessionUuid)
}
// Create matter record
dirRelativePath := dirMatter.Path
fileRelativePath := dirRelativePath + "/" + session.Filename
matter := &Matter{
Puuid: dirMatter.Uuid,
UserUuid: user.Uuid,
SpaceName: space.Name,
SpaceUuid: space.Uuid,
Dir: false,
Name: session.Filename,
Md5: session.FileMd5,
Size: totalWritten,
Privacy: session.Privacy,
Path: fileRelativePath,
Prop: EMPTY_JSON_MAP,
VisitTime: time.Now(),
}
matter = this.matterDao.Create(matter)
// Update session status
session.Status = UPLOAD_SESSION_STATUS_COMPLETED
session.MatterUuid = matter.Uuid
this.uploadSessionDao.Save(session)
// Clean up chunk files
go core.RunWithRecovery(func() {
this.cleanupChunks(space.Name, sessionUuid)
this.matterService.ComputeRouteSize(dirMatter.Uuid, user, space)
})
this.logger.Info("Upload session %s completed, matter uuid: %s", sessionUuid, matter.Uuid)
return matter
}
// cleanupChunks removes chunk files and records with improved atomicity
// First delete database records, then delete files (safer approach)
func (this *ChunkService) cleanupChunks(spaceName string, sessionUuid string) {
chunkDir := GetSessionChunkDir(spaceName, sessionUuid)
// First, remove chunk records from database
// This ensures that even if file deletion fails, we don't have orphaned DB records
this.uploadChunkDao.DeleteBySessionUuid(sessionUuid)
// Then remove chunk directory with retry mechanism
maxRetries := 3
for i := 0; i < maxRetries; i++ {
err := os.RemoveAll(chunkDir)
if err == nil {
this.logger.Info("Cleaned up chunks for session %s", sessionUuid)
return
}
this.logger.Error("failed to remove chunk dir %s (attempt %d/%d): %s", chunkDir, i+1, maxRetries, err.Error())
// Wait before retry (exponential backoff)
if i < maxRetries-1 {
time.Sleep(time.Duration(100*(i+1)) * time.Millisecond)
}
}
// If all retries failed, log a critical warning
this.logger.Error("CRITICAL: Failed to remove chunk dir %s after %d attempts. Manual cleanup may be required.", chunkDir, maxRetries)
}
// CancelSession cancels an upload session
func (this *ChunkService) CancelSession(sessionUuid string, user *User, space *Space) {
session := this.uploadSessionDao.CheckByUuid(sessionUuid)
// Validate ownership
if session.UserUuid != user.Uuid {
panic(result.BadRequest("session does not belong to you"))
}
// Clean up
this.cleanupChunks(space.Name, sessionUuid)
// Delete session
this.uploadSessionDao.Delete(session)
this.logger.Info("Cancelled upload session %s", sessionUuid)
}
// CleanExpiredSessions cleans up expired sessions
func (this *ChunkService) CleanExpiredSessions() {
sessions := this.uploadSessionDao.FindExpiredSessions()
for _, session := range sessions {
space := this.spaceDao.FindByUuid(session.SpaceUuid)
if space != nil {
this.cleanupChunks(space.Name, session.Uuid)
}
this.uploadSessionDao.Delete(session)
this.logger.Info("Cleaned expired session %s", session.Uuid)
}
}
+2
View File
@@ -81,6 +81,8 @@ func (this *InstallController) Init() {
&Space{},
&SpaceMember{},
&UploadToken{},
&UploadSession{},
&UploadChunk{},
&User{},
}
+30 -3
View File
@@ -1,15 +1,16 @@
package rest
import (
"math"
"os"
"time"
"github.com/eyebluecn/tank/code/core"
"github.com/eyebluecn/tank/code/tool/builder"
"github.com/eyebluecn/tank/code/tool/result"
"github.com/eyebluecn/tank/code/tool/util"
"github.com/eyebluecn/tank/code/tool/uuid"
"gorm.io/gorm"
"math"
"os"
"time"
)
type MatterDao struct {
@@ -311,6 +312,32 @@ func (this *MatterDao) FindBySpaceNameAndPuuidAndDirAndName(spaceName string, pu
return matter
}
func (this *MatterDao) FindByPuuidAndSpaceUuid(puuid string, spaceUuid string, sortArray []builder.OrderPair) []*Matter {
var matters []*Matter
var wp = &builder.WherePair{}
wp = wp.And(&builder.WherePair{Query: "puuid = ? AND space_uuid = ?", Args: []interface{}{puuid, spaceUuid}})
if sortArray == nil {
sortArray = []builder.OrderPair{
{
Key: "dir",
Value: DIRECTION_DESC,
},
{
Key: "create_time",
Value: DIRECTION_DESC,
},
}
}
db := core.CONTEXT.GetDB().Model(&Matter{}).Where(wp.Query, wp.Args...).Order(this.GetSortString(sortArray)).Find(&matters)
this.PanicError(db.Error)
return matters
}
func (this *MatterDao) FindByPuuidAndUserUuid(puuid string, userUuid string, sortArray []builder.OrderPair) []*Matter {
return this.FindByPuuidAndUserUuidAndDeleted(puuid, userUuid, "", sortArray)
}
+15 -16
View File
@@ -3,12 +3,6 @@ package rest
import (
"archive/zip"
"fmt"
"github.com/eyebluecn/tank/code/core"
"github.com/eyebluecn/tank/code/tool/builder"
"github.com/eyebluecn/tank/code/tool/download"
"github.com/eyebluecn/tank/code/tool/i18n"
"github.com/eyebluecn/tank/code/tool/result"
"github.com/eyebluecn/tank/code/tool/util"
"io"
"io/ioutil"
"mime/multipart"
@@ -19,6 +13,13 @@ import (
"regexp"
"strings"
"time"
"github.com/eyebluecn/tank/code/core"
"github.com/eyebluecn/tank/code/tool/builder"
"github.com/eyebluecn/tank/code/tool/download"
"github.com/eyebluecn/tank/code/tool/i18n"
"github.com/eyebluecn/tank/code/tool/result"
"github.com/eyebluecn/tank/code/tool/util"
)
/**
@@ -213,12 +214,12 @@ func (this *MatterService) DownloadZip(
if matters == nil || len(matters) == 0 {
panic(result.BadRequest("matters cannot be nil."))
}
userUuid := matters[0].UserUuid
spaceUuid := matters[0].SpaceUuid
puuid := matters[0].Puuid
for _, m := range matters {
if m.UserUuid != userUuid {
panic(result.BadRequest("userUuid not same"))
if m.SpaceUuid != spaceUuid {
panic(result.BadRequest("spaceUuid not same"))
} else if m.Puuid != puuid {
panic(result.BadRequest("puuid not same"))
}
@@ -285,13 +286,13 @@ func (this *MatterService) zipMatters(request *http.Request, matters []*Matter,
if matters == nil || len(matters) == 0 {
panic(result.BadRequest("matters cannot be nil."))
}
userUuid := matters[0].UserUuid
spaceUuid := matters[0].SpaceUuid
puuid := matters[0].Puuid
baseDirPath := util.GetDirOfPath(matters[0].AbsolutePath()) + "/"
for _, m := range matters {
if m.UserUuid != userUuid {
panic(result.BadRequest("userUuid not same"))
if m.SpaceUuid != spaceUuid {
panic(result.BadRequest("spaceUuid not same"))
} else if m.Puuid != puuid {
panic(result.BadRequest("puuid not same"))
}
@@ -482,9 +483,7 @@ func (this *MatterService) Upload(request *http.Request, file io.Reader, fileHea
panic(result.BadRequest("Dir has been deleted. Cannot upload under it."))
}
if len(filename) > MATTER_NAME_MAX_LENGTH {
panic(result.BadRequestI18n(request, i18n.MatterNameLengthExceedLimit, len(filename), MATTER_NAME_MAX_LENGTH))
}
CheckMatterName(request, filename)
//if fileHeader.Size not nill . check size in advance.
if fileHeader != nil {
@@ -1260,7 +1259,7 @@ func (this *MatterService) WrapChildrenDetail(request *http.Request, matter *Mat
if matter.Dir {
children := this.matterDao.FindByPuuidAndUserUuid(matter.Uuid, matter.UserUuid, nil)
children := this.matterDao.FindByPuuidAndSpaceUuid(matter.Uuid, matter.SpaceUuid, nil)
matter.Children = children
for _, child := range matter.Children {
+7
View File
@@ -89,6 +89,7 @@ func (this *PreferenceController) Edit(writer http.ResponseWriter, request *http
downloadDirMaxNumStr := request.FormValue("downloadDirMaxNum")
defaultTotalSizeLimitStr := request.FormValue("defaultTotalSizeLimit")
allowRegisterStr := request.FormValue("allowRegister")
allowChunkUploadStr := request.FormValue("allowChunkUpload")
deletedKeepDaysStr := request.FormValue("deletedKeepDays")
if name == "" {
@@ -140,6 +141,11 @@ func (this *PreferenceController) Edit(writer http.ResponseWriter, request *http
allowRegister = true
}
var allowChunkUpload = false
if allowChunkUploadStr == TRUE {
allowChunkUpload = true
}
preference := this.preferenceDao.Fetch()
oldDeletedKeepDays := preference.DeletedKeepDays
preference.Name = name
@@ -151,6 +157,7 @@ func (this *PreferenceController) Edit(writer http.ResponseWriter, request *http
preference.DownloadDirMaxNum = downloadDirMaxNum
preference.DefaultTotalSizeLimit = defaultTotalSizeLimit
preference.AllowRegister = allowRegister
preference.AllowChunkUpload = allowChunkUpload
preference.DeletedKeepDays = deletedKeepDays
preference = this.preferenceService.Save(preference)
+1
View File
@@ -19,6 +19,7 @@ type Preference struct {
DownloadDirMaxNum int64 `json:"downloadDirMaxNum" gorm:"type:bigint(20) not null;default:-1"`
DefaultTotalSizeLimit int64 `json:"defaultTotalSizeLimit" gorm:"type:bigint(20) not null;default:-1"`
AllowRegister bool `json:"allowRegister" gorm:"type:tinyint(1) not null;default:0"`
AllowChunkUpload bool `json:"allowChunkUpload" gorm:"type:tinyint(1) not null;default:0"`
PreviewConfig string `json:"previewConfig" gorm:"type:text"`
ScanConfig string `json:"scanConfig" gorm:"type:text"`
DeletedKeepDays int64 `json:"deletedKeepDays" gorm:"type:bigint(20) not null;default:7"`
+22
View File
@@ -15,6 +15,7 @@ type TaskService struct {
dashboardService *DashboardService
preferenceService *PreferenceService
matterService *MatterService
chunkService *ChunkService
userDao *UserDao
spaceDao *SpaceDao
@@ -45,6 +46,12 @@ func (this *TaskService) Init() {
if b, ok := b.(*MatterService); ok {
this.matterService = b
}
b = core.CONTEXT.GetBean(this.chunkService)
if b, ok := b.(*ChunkService); ok {
this.chunkService = b
}
b = core.CONTEXT.GetBean(this.userDao)
if b, ok := b.(*UserDao); ok {
this.userDao = b
@@ -94,6 +101,18 @@ func (this *TaskService) InitCleanDeletedMattersTask() {
this.logger.Info("[cron job] Everyday 01:00 Clean deleted matters.")
}
// init the clean chunk sessions task.
func (this *TaskService) InitCleanExpiredChunkSessions() {
expression := "0 2 * * *"
cronJob := cron.New()
_, err := cronJob.AddFunc(expression, this.chunkService.CleanExpiredSessions)
core.PanicError(err)
cronJob.Start()
this.logger.Info("[cron job] Everyday 02:00 Clean expired chunk sessions.")
}
// scan task.
func (this *TaskService) doScanTask() {
@@ -210,6 +229,9 @@ func (this *TaskService) Bootstrap() {
//load the clean deleted matters task.
this.InitCleanDeletedMattersTask()
//load the clean chunk sessions task.
this.InitCleanExpiredChunkSessions()
//load the scan task.
this.InitScanTask()
+25 -20
View File
@@ -1,6 +1,9 @@
package rest
import (
"sync"
"time"
"github.com/eyebluecn/tank/code/core"
"github.com/eyebluecn/tank/code/tool/cache"
"github.com/eyebluecn/tank/code/tool/result"
@@ -8,7 +11,6 @@ import (
"github.com/eyebluecn/tank/code/tool/uuid"
"net/http"
"os"
"time"
)
// @Service
@@ -19,8 +21,10 @@ type UserService struct {
spaceService *SpaceService
//file lock
//file lock (legacy, kept for compatibility)
locker *cache.Table
// matterLocker is used for atomic matter operations with retry support
matterLocker sync.Map
matterDao *MatterDao
matterService *MatterService
@@ -100,32 +104,33 @@ func (this *UserService) Init() {
this.locker = cache.NewTable()
}
// lock a user's operation. If lock, user cannot operate file.
// lock a user's operation with retry mechanism.
// This allows concurrent operations to wait for each other instead of immediately failing.
// Uses sync.Map for thread-safe atomic operations.
func (this *UserService) MatterLock(userUuid string) {
maxRetries := 30 // Maximum retry attempts (30 * 200ms = 6 seconds max wait)
retryInterval := 200 // Retry interval in milliseconds
cacheItem, err := this.locker.Value(userUuid)
if err != nil {
this.logger.Error("error while get cache" + err.Error())
for i := 0; i < maxRetries; i++ {
// Try to acquire lock atomically
if _, loaded := this.matterLocker.LoadOrStore(userUuid, true); !loaded {
// Successfully acquired the lock
return
}
// Lock is held by another operation, wait and retry
if i < maxRetries-1 {
time.Sleep(time.Duration(retryInterval) * time.Millisecond)
}
}
if cacheItem != nil && cacheItem.Data() != nil {
panic(result.BadRequest("file is being operating, retry later"))
}
duration := 12 * time.Hour
this.locker.Add(userUuid, duration, true)
// After all retries, still cannot acquire lock
panic(result.BadRequest("file is being operating, retry later"))
}
// unlock
func (this *UserService) MatterUnlock(userUuid string) {
exist := this.locker.Exists(userUuid)
if exist {
_, err := this.locker.Delete(userUuid)
this.PanicError(err)
} else {
this.logger.Error("unlock error. %s has no matter lock ", userUuid)
}
this.matterLocker.Delete(userUuid)
}
// load session to SessionCache. This method will be invoked in every request.
+6
View File
@@ -217,6 +217,12 @@ func (this *TankContext) registerBeans() {
//uploadToken
this.registerBean(new(rest.UploadTokenDao))
//chunk upload
this.registerBean(new(rest.ChunkController))
this.registerBean(new(rest.ChunkService))
this.registerBean(new(rest.UploadSessionDao))
this.registerBean(new(rest.UploadChunkDao))
//task
this.registerBean(new(rest.TaskService))
+31 -4
View File
@@ -46,16 +46,14 @@ func ExtractRequestArray(request *http.Request, key string) []string {
func ExtractRequestInt64(request *http.Request, key string) int64 {
keyStr := request.FormValue(key)
var num int64 = 0
if keyStr == "" {
panic(fmt.Sprintf("%s is required", key))
} else {
intVal, err := strconv.Atoi(keyStr)
intVal, err := strconv.ParseInt(keyStr, 10, 64)
if err != nil {
panic(err)
}
num = int64(intVal)
return num
return intVal
}
}
@@ -76,7 +74,36 @@ func ExtractRequestInt64WithErrorHint(request *http.Request, key string, errorHi
}
}
// param is optional. when missing, use default.
func ExtractRequestOptionalInt64(request *http.Request, key string, defaultValue int64) int64 {
str := request.FormValue(key)
if str == "" {
return defaultValue
} else {
intVal, err := strconv.ParseInt(str, 10, 64)
if err != nil {
panic(err)
}
return intVal
}
}
// param is required. when missing, panic error.
func ExtractRequestInt(request *http.Request, key string) int {
keyStr := request.FormValue(key)
if keyStr == "" {
panic(fmt.Sprintf("%s is required", key))
} else {
intVal, err := strconv.Atoi(keyStr)
if err != nil {
panic(err)
}
return intVal
}
}
// param is optional. when missing, use default.
func ExtractRequestOptionalInt(request *http.Request, key string, defaultValue int) int {
str := request.FormValue(key)
if str == "" {