diff --git a/backend/internal/controllers/about.go b/backend/internal/controllers/about.go index 9b818ed..a0ccb0b 100644 --- a/backend/internal/controllers/about.go +++ b/backend/internal/controllers/about.go @@ -11,7 +11,7 @@ import ( ) func GetAbout(c *echo.Context) error { - maxStorageSize, err := utils.GetFileSize(u.GetEnv("upload.maximum")) + maxStorageSize, err := u.GetFileSize(u.GetEnv("upload.maximum")) if err != nil { return utils.HTTPErrorHandler(c, err) } diff --git a/backend/internal/controllers/download.go b/backend/internal/controllers/download.go index a19f4c1..09ea53e 100644 --- a/backend/internal/controllers/download.go +++ b/backend/internal/controllers/download.go @@ -37,11 +37,11 @@ func DownloadShare(c *echo.Context) error { if shareInfo.Type == models.ShareTypeFile { fileInfo, _ := models.GetRedisFileInfo(shareInfo.Data) - uploadPath, err := utils.GetUploadDirPath() + uploadPath, err := u.GetUploadDirPath() if err != nil { return err } - return c.Attachment(fmt.Sprintf("%s/%s", uploadPath, utils.GetFileId(fileInfo.FileHash, fileInfo.FileSize)), shareInfo.FileName) + return c.Attachment(fmt.Sprintf("%s/%s", uploadPath, u.GetFileId(fileInfo.FileHash, fileInfo.FileSize)), shareInfo.FileName) } return utils.HTTPSuccessHandler(c, map[string]any{ "data": shareInfo.Data, diff --git a/backend/internal/controllers/file.go b/backend/internal/controllers/file.go index 7c2ebd1..63e5eb0 100644 --- a/backend/internal/controllers/file.go +++ b/backend/internal/controllers/file.go @@ -9,11 +9,12 @@ import ( "mime/multipart" "os" "pkg/models" + s "pkg/services" u "pkg/utils" "time" - "github.com/hibiken/asynq" "github.com/labstack/echo/v5" + "github.com/spf13/cast" ) func CreateUploadTask(c *echo.Context) error { @@ -26,14 +27,14 @@ func CreateUploadTask(c *echo.Context) error { if r.FileSize == 0 || r.MimeType == "" || r.FileHash == "" { return utils.HTTPErrorHandler(c, errors.New("调用接口参数错误")) } - fileId := utils.GetFileId(r.FileHash, r.FileSize) + fileId := u.GetFileId(r.FileHash, r.FileSize) fileInfo, err := models.GetRedisFileInfo(fileId) if err != nil { return utils.HTTPErrorHandler(c, err) } if fileInfo != nil { - uploadPath, err := utils.GetUploadDirPath() + uploadPath, err := u.GetUploadDirPath() if err != nil { return utils.HTTPErrorHandler(c, err) } @@ -52,7 +53,7 @@ func CreateUploadTask(c *echo.Context) error { "chunks": sliceList, }) } - maxStorageSize, err := utils.GetFileSize(u.GetEnv("upload.maximum")) + maxStorageSize, err := u.GetFileSize(u.GetEnv("upload.maximum")) if err != nil { return utils.HTTPErrorHandler(c, err) } @@ -78,7 +79,7 @@ func CreateUploadTask(c *echo.Context) error { for r.FileSize/ChunkSize > 1000 { ChunkSize *= 2 } - uploadTaskExpire := int64(3600) + uploadTaskExpire := cast.ToInt64(u.GetEnvWithDefault("upload.remove_expire", "2")) * 3600 newFileInfo := models.RedisFileInfo{ FileType: models.FileTypeInit, FileInfo: models.FileInfo{ @@ -95,14 +96,7 @@ func CreateUploadTask(c *echo.Context) error { return utils.HTTPErrorHandler(c, err) } - client := u.GetQueueClient() - json, err := json.Marshal(map[string]any{ - "file_id": fileId, - }) - if err != nil { - return utils.HTTPErrorHandler(c, err) - } - _, err = client.Enqueue(asynq.NewTask("file:remove", json), asynq.ProcessIn(time.Duration(uploadTaskExpire)*time.Second)) + err = s.SetFileRemoveTask(fileId, time.Duration(uploadTaskExpire)*time.Second) if err != nil { return utils.HTTPErrorHandler(c, err) } @@ -161,7 +155,7 @@ func UploadFileSlice(c *echo.Context) error { } defer file.Close() - uploadPath, err := utils.GetUploadDirPath() + uploadPath, err := u.GetUploadDirPath() if err != nil { return utils.HTTPErrorHandler(c, err) } @@ -203,7 +197,7 @@ func FinishUploadTask(c *echo.Context) error { return utils.HTTPErrorHandler(c, errors.New("上传任务已过期")) } - uploadPath, err := utils.GetUploadDirPath() + uploadPath, err := u.GetUploadDirPath() if err != nil { return utils.HTTPErrorHandler(c, err) } @@ -231,7 +225,7 @@ func FinishUploadTask(c *echo.Context) error { return utils.HTTPErrorHandler(c, err) } - file_hash, err := utils.GetFileMd5(file) + file_hash, err := u.GetFileMd5(file) if err != nil { file.Close() diff --git a/backend/internal/utils/file.go b/backend/internal/utils/file.go deleted file mode 100644 index 0be6073..0000000 --- a/backend/internal/utils/file.go +++ /dev/null @@ -1,56 +0,0 @@ -package utils - -import ( - "bufio" - "crypto/md5" - "fmt" - "io" - "os" - "path/filepath" - "pkg/utils" - - humanize "github.com/dustin/go-humanize" -) - -func GetFileId(fileHash string, fileSize int64) string { - return fmt.Sprintf("%s_%d", fileHash, fileSize) -} - -func GetFileMd5(file io.Reader) (string, error) { - - const bufferSize = 1024 * 1000 // 1MB - - hash := md5.New() - for buf, reader := make([]byte, bufferSize), bufio.NewReader(file); ; { - n, err := reader.Read(buf) - if err != nil { - if err == io.EOF { - break - } - return "", err - } - - hash.Write(buf[:n]) - } - - return fmt.Sprintf("%x", hash.Sum(nil)), nil -} - -func GetUploadDirPath() (string, error) { - uploadPath := utils.GetEnv("upload.path") - if uploadPath == "" { - basepath, err := os.Getwd() - if err != nil { - return "", err - } - uploadPath = filepath.Join(basepath, "uploads") - } - if err := os.MkdirAll(uploadPath, 0755); err != nil { - return "", err - } - return uploadPath, nil -} - -func GetFileSize(size string) (uint64, error) { - return humanize.ParseBytes(size) -} diff --git a/go.work b/go.work index 8a1e812..8839c1b 100644 --- a/go.work +++ b/go.work @@ -3,6 +3,7 @@ go 1.25.5 use ( ./backend ./pkg/models + ./pkg/services ./pkg/utils ./worker ) diff --git a/go.work.sum b/go.work.sum index b4adc4d..fbba3d7 100644 --- a/go.work.sum +++ b/go.work.sum @@ -4,10 +4,13 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38= github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bmatcuk/doublestar/v4 v4.9.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/casbin/casbin/v2 v2.105.0 h1:dLj5P6pLApBRat9SADGiLxLZjiDPvA1bsPkyV4PGx6I= github.com/casbin/casbin/v2 v2.105.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/casbin/v2 v2.135.0/go.mod h1:FmcfntdXLTcYXv/hxgNntcRPqAbwOG9xsism0yXT+18= github.com/casbin/govaluate v1.3.0 h1:VA0eSY0M2lA86dYd5kPPuNZMUD9QkWnOCnavGrw9myc= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= +github.com/casbin/govaluate v1.10.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= @@ -23,12 +26,15 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -49,6 +55,7 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= @@ -68,6 +75,7 @@ golang.org/x/telemetry v0.0.0-20250807160809-1a19826ec488/go.mod h1:fGb/2+tgXXjh golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= @@ -78,5 +86,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IV golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/services/file.go b/pkg/services/file.go new file mode 100644 index 0000000..764e0ad --- /dev/null +++ b/pkg/services/file.go @@ -0,0 +1,22 @@ +package services + +import ( + "encoding/json" + "time" + + "pkg/utils" + + "github.com/hibiken/asynq" +) + +func SetFileRemoveTask(fileId string, expire time.Duration) error { + client := utils.GetQueueClient() + json, err := json.Marshal(map[string]any{ + "file_id": fileId, + }) + if err != nil { + return err + } + _, err = client.Enqueue(asynq.NewTask("file:remove", json), asynq.ProcessIn(time.Duration(expire)*time.Second)) + return err +} diff --git a/pkg/services/go.mod b/pkg/services/go.mod new file mode 100644 index 0000000..521c39e --- /dev/null +++ b/pkg/services/go.mod @@ -0,0 +1,3 @@ +module pkg/services + +go 1.25.5 diff --git a/worker/internal/services/file.go b/worker/internal/services/file.go index a928165..1e63a45 100644 --- a/worker/internal/services/file.go +++ b/worker/internal/services/file.go @@ -5,8 +5,11 @@ import ( "os" "path/filepath" "pkg/models" + "pkg/services" + u "pkg/utils" "time" - "worker/internal/utils" + + "github.com/spf13/cast" ) type GenStandardFileReturn struct { @@ -31,14 +34,14 @@ func GenStandardFile(filePath string, mimeType string) (GenStandardFileReturn, e } fileSize := fileInfo.Size() - fileHash, err := utils.GetFileMd5(file) + fileHash, err := u.GetFileMd5(file) if err != nil { return GenStandardFileReturn{}, err } - fileId := utils.GetFileId(fileHash, fileSize) + fileId := u.GetFileId(fileHash, fileSize) - uploadPath, err := utils.GetUploadDirPath() + uploadPath, err := u.GetUploadDirPath() if err != nil { return GenStandardFileReturn{}, err } @@ -55,7 +58,11 @@ func GenStandardFile(filePath string, mimeType string) (GenStandardFileReturn, e FileType: models.FileTypeUpload, CreatedAt: time.Now().Unix(), }) - + expire := cast.ToInt64(u.GetEnvWithDefault("upload.remove_expire", "2")) * 3600 + err = services.SetFileRemoveTask(fileId, time.Duration(expire)*time.Second) + if err != nil { + return GenStandardFileReturn{}, err + } return GenStandardFileReturn{ FileId: fileId, FileInfo: models.FileInfo{ diff --git a/worker/internal/utils/file.go b/worker/internal/utils/file.go deleted file mode 100644 index 5c4819a..0000000 --- a/worker/internal/utils/file.go +++ /dev/null @@ -1,65 +0,0 @@ -package utils - -import ( - "bufio" - "crypto/md5" - "fmt" - "io" - "os" - "path/filepath" - "pkg/utils" -) - -func GetFileId(fileHash string, fileSize int64) string { - return fmt.Sprintf("%s_%d", fileHash, fileSize) -} - -func GetFileMd5(file io.Reader) (string, error) { - - const bufferSize = 1024 * 1000 // 1MB - - hash := md5.New() - for buf, reader := make([]byte, bufferSize), bufio.NewReader(file); ; { - n, err := reader.Read(buf) - if err != nil { - if err == io.EOF { - break - } - return "", err - } - - hash.Write(buf[:n]) - } - - return fmt.Sprintf("%x", hash.Sum(nil)), nil -} - -func GetUploadDirPath() (string, error) { - basepath, err := os.Getwd() - if err != nil { - return "", err - } - finalPath := filepath.Join(basepath, "uploads") - uploadPath := utils.GetEnvWithDefault("upload.path", finalPath) - if err := os.MkdirAll(uploadPath, 0755); err != nil { - return "", err - } - return uploadPath, nil -} - -func CopyFile(src, dst string) error { - sourceFile, err := os.Open(src) - if err != nil { - return err - } - defer sourceFile.Close() - - destFile, err := os.Create(dst) - if err != nil { - return err - } - defer destFile.Close() - - _, err = io.Copy(destFile, sourceFile) - return err -}