Finish the oss uploading feature.

CHANGELOG

@@ -47,6 +47,10 @@ c. 在github上发布新版本。
 
 3.更新tank-doc 下载内容
 
+tank-3.1.3 2021-08-27
+1.tank.json中新增5个字段。
+
+
 tank-3.1.2 2021-01-02
 1. Add scan manually once feature
 2. Add disable recycle bin feature

code/core/config.go

@@ -22,6 +22,13 @@ type Config interface {
 	MysqlUrl() string
 	//files storage location.
 	MatterPath() string
+
+	OssAccessKey() string
+	OssSecretKey() string
+	OssEndpoint() string
+	OssCustomHost() string
+	OssBucket() string
+
 	//when installed by user. Write configs to tank.json
 	FinishInstall(mysqlPort int, mysqlHost string, mysqlSchema string, mysqlUsername string, mysqlPassword string, mysqlCharset string)
 }

code/rest/matter_controller.go

@@ -5,6 +5,7 @@ import (
 	"github.com/eyebluecn/tank/code/tool/builder"
 	"github.com/eyebluecn/tank/code/tool/i18n"
 	"github.com/eyebluecn/tank/code/tool/result"
+	"math"
 	"net/http"
 	"strconv"
 	"strings"
@@ -21,6 +22,9 @@ type MatterController struct {
 	shareService      *ShareService
 	bridgeDao         *BridgeDao
 	imageCacheService *ImageCacheService
+	ossService        *OssService
+
+	ossUploading bool
 }
 
 func (this *MatterController) Init() {
@@ -65,6 +69,12 @@ func (this *MatterController) Init() {
 	if b, ok := b.(*ImageCacheService); ok {
 		this.imageCacheService = b
 	}
+	b = core.CONTEXT.GetBean(this.ossService)
+	if b, ok := b.(*OssService); ok {
+		this.ossService = b
+	}
+
+	this.ossUploading = false
 }
 
 func (this *MatterController) RegisterRoutes() map[string]func(writer http.ResponseWriter, request *http.Request) {
@@ -91,6 +101,9 @@ func (this *MatterController) RegisterRoutes() map[string]func(writer http.Respo
 	routeMap["/api/matter/mirror"] = this.Wrap(this.Mirror, USER_ROLE_USER)
 	routeMap["/api/matter/zip"] = this.Wrap(this.Zip, USER_ROLE_USER)
 
+	routeMap["/api/matter/oss"] = this.Wrap(this.Oss, USER_ROLE_ADMINISTRATOR)
+	routeMap["/api/matter/stop"] = this.Wrap(this.Stop, USER_ROLE_ADMINISTRATOR)
+
 	return routeMap
 }
 
@@ -625,3 +638,56 @@ func (this *MatterController) Zip(writer http.ResponseWriter, request *http.Requ
 
 	return nil
 }
+
+//upload to oss.
+func (this *MatterController) Oss(writer http.ResponseWriter, request *http.Request) *result.WebResult {
+
+	if this.ossUploading {
+		return this.Success("oss is running")
+	} else {
+		this.ossUploading = true
+	}
+
+	pageSize := 100
+	count, _ := this.matterDao.PlainPage(0, pageSize, "", "", "", "false", "", nil, nil, nil)
+	if count > 0 {
+		var totalPages = int(math.Ceil(float64(count) / float64(pageSize)))
+
+		var page int
+		for page = 0; page < totalPages; page++ {
+
+			this.logger.Info("-- processing %d/%d", page+1, totalPages)
+
+			if !this.ossUploading {
+				break
+			}
+
+			_, matters := this.matterDao.PlainPage(page, pageSize, "", "", "", "false", "", nil, nil, nil)
+			for _, matter := range matters {
+
+				if !this.ossUploading {
+					break
+				}
+
+				//保证不抛异常
+				core.RunWithRecovery(func() {
+					this.ossService.Upload(matter)
+				})
+
+			}
+		}
+	}
+
+	this.ossUploading = false
+
+	return this.Success(nil)
+}
+
+//stop uploading to oss.
+func (this *MatterController) Stop(writer http.ResponseWriter, request *http.Request) *result.WebResult {
+
+	this.ossUploading = false
+
+	this.logger.Info("-- stop processing --")
+	return this.Success("OK")
+}

code/rest/matter_dao.go

@@ -355,7 +355,7 @@ func (this *MatterDao) PageHandle(
 
 		var page int
 		for page = 0; page < totalPages; page++ {
-			_, matters := this.PlainPage(0, pageSize, puuid, userUuid, name, dir, deleted, deleteTimeBefore, nil, sortArray)
+			_, matters := this.PlainPage(page, pageSize, puuid, userUuid, name, dir, deleted, deleteTimeBefore, nil, sortArray)
 			for _, matter := range matters {
 				fun(matter)
 			}

code/rest/matter_model.go

@@ -36,7 +36,7 @@ type Matter struct {
 	Username   string    `json:"username" gorm:"type:varchar(45) not null"`
 	Dir        bool      `json:"dir" gorm:"type:tinyint(1) not null;default:0"`
 	Name       string    `json:"name" gorm:"type:varchar(255) not null"`
-	Md5        string    `json:"md5" gorm:"type:varchar(45)"`
+	Md5        string    `json:"md5" gorm:"type:varchar(512)"`
 	Size       int64     `json:"size" gorm:"type:bigint(20) not null;default:0"`
 	Privacy    bool      `json:"privacy" gorm:"type:tinyint(1) not null;default:0"`
 	Path       string    `json:"path" gorm:"type:varchar(1024)"`

code/rest/oss_service.go

@@ -0,0 +1,81 @@
+package rest
+
+import (
+	"fmt"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"github.com/eyebluecn/tank/code/core"
+)
+
+// oss service
+//@Service
+type OssService struct {
+	BaseBean
+	matterDao *MatterDao
+
+	//whether scan task is running
+	client *oss.Client
+	bucket *oss.Bucket
+}
+
+func (this *OssService) Init() {
+	this.BaseBean.Init()
+
+	b := core.CONTEXT.GetBean(this.matterDao)
+	if b, ok := b.(*MatterDao); ok {
+		this.matterDao = b
+	}
+
+}
+
+//upload a matter to oss.
+func (this *OssService) Upload(matter *Matter) {
+
+	//只上传文件，忽略文件夹。
+	if matter.Dir {
+		return
+	}
+
+	//只上传没有传过的。
+	if matter.Md5 != "" {
+		return
+	}
+
+	// <yourObjectName>上传文件到OSS时需要指定包含文件后缀在内的完整路径，例如abc/efg/123.jpg。
+	objectName := matter.Path
+	if len(matter.Path) > 1 {
+		objectName = matter.Path[1:]
+	}
+
+	// <yourLocalFileName>由本地文件路径加文件名包括后缀组成，例如/users/local/myfile.txt。
+	localFileName := matter.AbsolutePath()
+
+	// 上传文件。
+	err := this.bucket.PutObjectFromFile(objectName, localFileName)
+	this.PanicError(err)
+
+	// 修改本地文件的md5字段
+	matter.Md5 = fmt.Sprintf("%s/%s", core.CONFIG.OssCustomHost(), objectName)
+	this.matterDao.Save(matter)
+
+	this.logger.Info("upload %s %s to oss path = %s", matter.Uuid, matter.Name, objectName)
+
+}
+
+func (this *OssService) Bootstrap() {
+
+	// Endpoint以杭州为例，其它Region请按实际情况填写。
+	// 阿里云主账号AccessKey拥有所有API的访问权限，风险很高。强烈建议您创建并使用RAM账号进行API访问或日常运维，请登录 https://ram.console.aliyun.com 创建RAM账号。
+
+	// 创建OSSClient实例。
+	client, err := oss.New(fmt.Sprintf("http://%s", core.CONFIG.OssEndpoint()), core.CONFIG.OssAccessKey(), core.CONFIG.OssSecretKey())
+	this.client = client
+	this.PanicError(err)
+
+	// 获取存储空间。
+	bucket, err := this.client.Bucket(core.CONFIG.OssBucket())
+	this.bucket = bucket
+	this.PanicError(err)
+
+	this.logger.Info("OssService bootstrap.")
+
+}

code/support/tank_config.go

@@ -42,6 +42,12 @@ type ConfigItem struct {
 	MysqlPassword string
 	//mysql charset
 	MysqlCharset string
+
+	OssAccessKey  string
+	OssSecretKey  string
+	OssEndpoint   string
+	OssCustomHost string
+	OssBucket     string
 }
 
 //validate whether the config file is ok
@@ -81,6 +87,30 @@ func (this *ConfigItem) validate() bool {
 		return false
 	}
 
+	if this.OssAccessKey == "" {
+		core.LOGGER.Error("OssAccessKey  is not configured")
+		return false
+	}
+
+	if this.OssSecretKey == "" {
+		core.LOGGER.Error("OssSecretKey  is not configured")
+		return false
+	}
+
+	if this.OssEndpoint == "" {
+		core.LOGGER.Error("OssEndpoint  is not configured")
+		return false
+	}
+
+	if this.OssCustomHost == "" {
+		core.LOGGER.Error("OssCustomHost  is not configured")
+		return false
+	}
+	if this.OssBucket == "" {
+		core.LOGGER.Error("OssBucket  is not configured")
+		return false
+	}
+
 	return true
 
 }
@@ -178,6 +208,26 @@ func (this *TankConfig) MatterPath() string {
 	return this.matterPath
 }
 
+func (this *TankConfig) OssAccessKey() string {
+	return this.item.OssAccessKey
+}
+
+func (this *TankConfig) OssSecretKey() string {
+	return this.item.OssSecretKey
+}
+
+func (this *TankConfig) OssEndpoint() string {
+	return this.item.OssEndpoint
+}
+
+func (this *TankConfig) OssCustomHost() string {
+	return this.item.OssCustomHost
+}
+
+func (this *TankConfig) OssBucket() string {
+	return this.item.OssBucket
+}
+
 //Finish the installation. Write config to tank.json
 func (this *TankConfig) FinishInstall(mysqlPort int, mysqlHost string, mysqlSchema string, mysqlUsername string, mysqlPassword string, mysqlCharset string) {
 

code/support/tank_context.go

@@ -147,6 +147,9 @@ func (this *TankContext) registerBeans() {
 	this.registerBean(new(rest.MatterDao))
 	this.registerBean(new(rest.MatterService))
 
+	//oss
+	this.registerBean(new(rest.OssService))
+
 	//preference
 	this.registerBean(new(rest.PreferenceController))
 	this.registerBean(new(rest.PreferenceDao))

go.mod

@@ -3,6 +3,7 @@ module github.com/eyebluecn/tank
 go 1.14
 
 require (
+	github.com/aliyun/aliyun-oss-go-sdk v2.1.10+incompatible
 	github.com/disintegration/imaging v1.6.2
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/jinzhu/gorm v1.9.14
@@ -10,4 +11,5 @@ require (
 	github.com/robfig/cron/v3 v3.0.1
 	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
 	golang.org/x/text v0.3.3
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 )

go.sum

@@ -1,4 +1,6 @@
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/aliyun/aliyun-oss-go-sdk v2.1.10+incompatible h1:D3gwOr9qUUmyyBRDbpnATqu+EkqqmigFd3Od6xO1QUU=
+github.com/aliyun/aliyun-oss-go-sdk v2.1.10+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -57,4 +59,6 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

main.go

@@ -1,13 +1,15 @@
 package main
 
 import (
+	"fmt"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"github.com/eyebluecn/tank/code/core"
 	"github.com/eyebluecn/tank/code/support"
 	_ "github.com/go-sql-driver/mysql"
 )
 
 func main() {
-
+	fmt.Println("OSS Go SDK Version: ", oss.Version)
 	core.APPLICATION = &support.TankApplication{}
 	core.APPLICATION.Start()