diff --git a/pkg/models/go.mod b/pkg/models/go.mod index 151bfb1..2b7df42 100644 --- a/pkg/models/go.mod +++ b/pkg/models/go.mod @@ -2,7 +2,10 @@ module pkg/models go 1.25.5 -require github.com/redis/rueidis v1.0.73 +require ( + github.com/redis/rueidis v1.0.73 + github.com/spf13/cast v1.10.0 +) require ( golang.org/x/net v0.52.0 // indirect diff --git a/pkg/models/go.sum b/pkg/models/go.sum index 359e460..d925dba 100644 --- a/pkg/models/go.sum +++ b/pkg/models/go.sum @@ -1,9 +1,14 @@ +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/onsi/gomega v1.38.3 h1:eTX+W6dobAYfFeGC2PV6RwXRu/MyT+cQguijutvkpSM= github.com/onsi/gomega v1.38.3/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/redis/rueidis v1.0.73 h1:0Enrg0VuMdaYyNDDj0lLIheWY0uybCeQOh+jTp2GG3M= github.com/redis/rueidis v1.0.73/go.mod h1:lfdcZzJ1oKGKL37vh9fO3ymwt+0TdjkkUCJxbgpmcgQ= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= diff --git a/worker/go.mod b/worker/go.mod index 1fbb124..85cb2f7 100644 --- a/worker/go.mod +++ b/worker/go.mod @@ -3,6 +3,7 @@ module worker go 1.25.5 require ( + github.com/go-resty/resty/v2 v2.16.5 github.com/google/uuid v1.6.0 github.com/hibiken/asynq v0.26.0 github.com/samber/lo v1.53.0 @@ -15,9 +16,14 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/openai/openai-go/v3 v3.30.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/redis/go-redis/v9 v9.18.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/sys v0.42.0 // indirect diff --git a/worker/go.sum b/worker/go.sum index b67c3a7..b3f5dcf 100644 --- a/worker/go.sum +++ b/worker/go.sum @@ -22,6 +22,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/openai/openai-go/v3 v3.30.0 h1:T8VkhqAm6BuvxwpVG+Aw+H4TcYIsbj9nqytjpWcE/aU= +github.com/openai/openai-go/v3 v3.30.0/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= @@ -36,6 +38,17 @@ github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= diff --git a/worker/internal/services/llm/client.go b/worker/internal/services/llm/client.go new file mode 100644 index 0000000..4d86b0c --- /dev/null +++ b/worker/internal/services/llm/client.go @@ -0,0 +1,56 @@ +package llm + +import ( + "context" + "errors" + + "github.com/openai/openai-go/v3" + "github.com/openai/openai-go/v3/option" + "github.com/openai/openai-go/v3/responses" +) + +var ErrNotConfigured = errors.New("LLMNotConfigured") +var client openai.Client + +// NewClient 从配置读取 llm.endpoint / llm.api_key / llm.model 并返回客户端。 +// 若 endpoint 或 api_key 未配置则返回 ErrNotConfigured。 +func NewClient(endpoint, apiKey string) (openai.Client, error) { + if endpoint == "" || apiKey == "" { + return openai.Client{}, ErrNotConfigured + } + client = openai.NewClient(option.WithBaseURL(endpoint), option.WithAPIKey(apiKey)) + return client, nil +} + +// Chat 发送 system + user 消息对，返回模型回复的文本内容。 +func Chat(ctx context.Context, systemPrompt, userPrompt string, model string) (string, error) { + resp, err := client.Responses.New(ctx, responses.ResponseNewParams{ + Instructions: openai.String(systemPrompt), + Input: responses.ResponseNewParamsInputUnion{ + OfString: openai.String(userPrompt), + }, + Model: model, + }) + + if err != nil { + return "", err + } + return resp.OutputText(), nil +} + +// 流式获取 +func ChatWithStream(ctx context.Context, systemPrompt, userPrompt string, handler func(event responses.ResponseStreamEventUnion)) error { + stream := client.Responses.NewStreaming(ctx, responses.ResponseNewParams{ + Input: responses.ResponseNewParamsInputUnion{OfString: openai.String(userPrompt)}, + Model: openai.ChatModelGPT5_2, + }) + for stream.Next() { + event := stream.Current() + handler(event) + } + + if stream.Err() != nil { + return stream.Err() + } + return nil +} diff --git a/worker/internal/services/text.go b/worker/internal/services/text.go new file mode 100644 index 0000000..09b3ecd --- /dev/null +++ b/worker/internal/services/text.go @@ -0,0 +1,208 @@ +package services + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "pkg/utils" + "strings" + "worker/internal/services/llm" + + "github.com/go-resty/resty/v2" +) + +var ( + ErrProviderNotConfigured = errors.New("ProviderNotConfigured") + ErrUnknownProvider = errors.New("UnknownProvider") +) + +const microsoftAuthEndpoint = "https://edge.microsoft.com/translate/auth" + +func TranslateText(text, from, to, provider string) (string, error) { + switch provider { + case "google": + return translateWithGoogle(text, from, to) + case "microsoft": + return translateWithMicrosoft(text, from, to) + case "deeplx": + return translateWithDeepLX(text, from, to) + case "deepseek": + return translateWithLLM(text, from, to) + default: + return "", fmt.Errorf("%w: %s", ErrUnknownProvider, provider) + } +} + +// translateWithGoogle 调用 Google Translate 非官方 API。 +// 响应格式：[[["翻译结果","原文",...]], null, "en"] +func translateWithGoogle(text, from, to string) (string, error) { + endpoint := fmt.Sprintf( + "https://translate.googleapis.com/translate_a/single?client=gtx&sl=%s&tl=%s&dt=t&q=%s", + url.QueryEscape(from), url.QueryEscape(to), url.QueryEscape(text), + ) + resp, err := http.Get(endpoint) //nolint:noctx + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + var raw []any + if err := json.Unmarshal(body, &raw); err != nil { + return "", err + } + if len(raw) == 0 { + return "", errors.New("GoogleTranslateUnexpectedResponse") + } + segments, ok := raw[0].([]any) + if !ok { + return "", errors.New("GoogleTranslateUnexpectedResponse") + } + var parts []string + for _, seg := range segments { + pair, ok := seg.([]any) + if !ok || len(pair) == 0 { + continue + } + if s, ok := pair[0].(string); ok && s != "" { + parts = append(parts, s) + } + } + return strings.Join(parts, ""), nil +} + +// translateWithMicrosoft 调用微软 Edge Translator 免费端点（无需 API Key）。 +// 响应格式：[{"translations":[{"text":"翻译结果","to":"zh-Hans"}]}] +func translateWithMicrosoft(text, from, to string) (string, error) { + params := url.Values{} + params.Set("api-version", "3.0") + params.Set("to", to) + if from != "auto" { + params.Set("from", from) + } + endpoint := "https://api-edge.cognitive.microsofttranslator.com/translate?" + params.Encode() + + reqBody, err := json.Marshal([]map[string]string{{"Text": text}}) + if err != nil { + return "", err + } + token, err := getMicrosoftToken() + if err != nil { + return "", err + } + + client := resty.New() + req := client.R(). + SetHeader("Content-Type", "application/json"). + SetBody(reqBody) + + req.SetAuthToken(token) + + resp, err := req.Post(endpoint) + if err != nil { + return "", err + } + if resp == nil { + return "", errors.New("MicrosoftTranslateUnexpectedResponse") + } + if resp.IsError() { + return "", fmt.Errorf("MicrosoftTranslateRequestFailed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + + body := resp.Body() + if len(body) == 0 { + return "", errors.New("MicrosoftTranslateUnexpectedResponse") + } + + var result []struct { + Translations []struct { + Text string `json:"text"` + } `json:"translations"` + } + if err := json.Unmarshal(body, &result); err != nil { + return "", err + } + if len(result) == 0 || len(result[0].Translations) == 0 { + return "", errors.New("MicrosoftTranslateUnexpectedResponse") + } + return result[0].Translations[0].Text, nil +} + +func getMicrosoftToken() (string, error) { + resp, err := resty.New().R().Get(microsoftAuthEndpoint) + if err != nil { + return "", err + } + if resp == nil { + return "", errors.New("MicrosoftAuthUnexpectedResponse") + } + if resp.IsError() { + return "", fmt.Errorf("MicrosoftAuthRequestFailed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + + token := strings.TrimSpace(resp.String()) + if token == "" { + return "", errors.New("MicrosoftAuthUnexpectedResponse") + } + return token, nil +} + +type deeplxRequest struct { + Text string `json:"text"` + SourceLang string `json:"source_lang"` + TargetLang string `json:"target_lang"` +} + +type deeplxResponse struct { + Data string `json:"data"` +} + +// translateWithDeepLX 调用自托管 DeepLX 服务，端点通过 deeplx.endpoint 配置。 +func translateWithDeepLX(text, from, to string) (string, error) { + endpoint := utils.GetEnv("deeplx.endpoint") + if endpoint == "" { + return "", ErrProviderNotConfigured + } + reqBody, err := json.Marshal(deeplxRequest{Text: text, SourceLang: from, TargetLang: to}) + if err != nil { + return "", err + } + resp, err := http.Post(endpoint, "application/json", strings.NewReader(string(reqBody))) //nolint:noctx + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + var result deeplxResponse + if err := json.Unmarshal(body, &result); err != nil { + return "", err + } + return result.Data, nil +} + +// translateWithLLM 通过 OpenAI 兼容 API（如 DeepSeek）进行 AI 翻译。 +func translateWithLLM(text, from, to string) (string, error) { + endpoint := utils.GetEnv("llm.endpoint") + apiKey := utils.GetEnv("llm.api_key") + _, err := llm.NewClient(endpoint, apiKey) + if err != nil { + return "", ErrProviderNotConfigured + } + model := utils.GetEnvWithDefault("llm.model", "deepseek-chat") + system := fmt.Sprintf( + "You are a professional translator. Translate the given text from %s to %s. "+ + "Output only the translated text, with no explanations or surrounding quotes.", + from, to, + ) + return llm.Chat(context.Background(), system, text, model) +} diff --git a/worker/internal/tasks/text.go b/worker/internal/tasks/text.go new file mode 100644 index 0000000..a7628c2 --- /dev/null +++ b/worker/internal/tasks/text.go @@ -0,0 +1,33 @@ +package tasks + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "pkg/models" + "worker/internal/services" + + "github.com/hibiken/asynq" +) + +func TranslateText(ctx context.Context, task *asynq.Task) error { + var payload TranslateTextTaskPayload + if err := json.Unmarshal(task.Payload(), &payload); err != nil { + return err + } + + result, err := services.TranslateText(payload.Text, payload.Source, payload.Target, payload.Provider) + if err != nil { + // 配置缺失或未知提供商属于永久性错误，跳过重试 + if errors.Is(err, services.ErrProviderNotConfigured) || errors.Is(err, services.ErrUnknownProvider) { + return fmt.Errorf("%w: %w", err, asynq.SkipRetry) + } + return err + } + + return models.SetRedisTaskInfo(task.ResultWriter().TaskID(), map[string]any{ + "status": "success", + "result": result, + }) +} diff --git a/worker/test/services/text_test.go b/worker/test/services/text_test.go new file mode 100644 index 0000000..d3489a2 --- /dev/null +++ b/worker/test/services/text_test.go @@ -0,0 +1,72 @@ +package services + +import ( + "testing" + "worker/internal/services" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// --- Google Translate --- + +func TestTranslateWithGoogle_EnToZhCN(t *testing.T) { + result, err := services.TranslateText("Hello world", "auto", "zh-CN", "google") + require.NoError(t, err) + assert.NotEmpty(t, result) + t.Logf("en→zh-CN: %s", result) +} + +func TestTranslateWithGoogle_ZhCNToEn(t *testing.T) { + result, err := services.TranslateText("你好世界", "zh-CN", "en", "google") + require.NoError(t, err) + assert.NotEmpty(t, result) + t.Logf("zh-CN→en: %s", result) +} + +func TestTranslateWithGoogle_AutoDetect(t *testing.T) { + result, err := services.TranslateText("こんにちは", "auto", "en", "google") + require.NoError(t, err) + assert.NotEmpty(t, result) + t.Logf("auto(ja)→en: %s", result) +} + +func TestTranslateWithGoogle_LongText(t *testing.T) { + text := "The quick brown fox jumps over the lazy dog. " + + "This sentence contains every letter of the English alphabet. " + + "It is commonly used for testing purposes." + result, err := services.TranslateText(text, "en", "zh-CN", "google") + require.NoError(t, err) + assert.NotEmpty(t, result) + t.Logf("长文本→zh-CN: %s", result) +} + +// --- Microsoft Translator --- + +func TestTranslateWithMicrosoft_EnToZhCN(t *testing.T) { + result, err := services.TranslateText("Hello world", "auto", "zh-Hans", "microsoft") + require.NoError(t, err) + assert.NotEmpty(t, result) + t.Logf("en→zh-Hans: %s", result) +} + +func TestTranslateWithMicrosoft_ZhCNToEn(t *testing.T) { + result, err := services.TranslateText("你好世界", "zh-Hans", "en", "microsoft") + require.NoError(t, err) + assert.NotEmpty(t, result) + t.Logf("zh-Hans→en: %s", result) +} + +func TestTranslateWithMicrosoft_AutoDetect(t *testing.T) { + result, err := services.TranslateText("こんにちは", "auto", "en", "microsoft") + require.NoError(t, err) + assert.NotEmpty(t, result) + t.Logf("auto(ja)→en: %s", result) +} + +// --- 通用 --- + +func TestTranslateText_UnknownProvider(t *testing.T) { + _, err := services.TranslateText("hello", "en", "zh-CN", "unknown_provider") + assert.ErrorIs(t, err, services.ErrUnknownProvider) +}