From ce0fa7f2edb5c438c9a2e761eadd108433b776bd Mon Sep 17 00:00:00 2001 From: hu xiaotong <416314413@163.com> Date: Fri, 18 Jul 2025 16:56:36 +0800 Subject: [PATCH] =?UTF-8?q?refactor(internal):=20=E4=BC=98=E5=8C=96=20OSS?= =?UTF-8?q?=20=E9=A2=84=E7=AD=BE=E5=90=8D=20URL=20=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E5=88=B7=E6=96=B0=E4=BB=BB=E5=8A=A1=E5=92=8C=E8=8B=B1=E9=9B=84?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=BC=93=E5=AD=98=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 注释掉 OSS预签名 URL 缓存刷新任务的定时执行代码 - 在 hero/hero.go 中增加对 Redis缓存和英雄数据集的非空校验 - 修改 OSS预签名 URL 生成逻辑，自动替换为 CDN 域名 --- internal/consts/consts.go | 2 + internal/logic/cron/third_party_sync.go | 223 ++++++++++++++++++++++-- internal/logic/hero/hero.go | 2 +- manifest/config/config.yaml | 2 +- 4 files changed, 210 insertions(+), 19 deletions(-) diff --git a/internal/consts/consts.go b/internal/consts/consts.go index a9dac83..1cc9613 100644 --- a/internal/consts/consts.go +++ b/internal/consts/consts.go @@ -18,6 +18,8 @@ const ( // 角色图片基础 URL GfHeroPngURL = "https://static.smilegatemegaport.com/event/live/epic7/guide/images/hero/" + EPIC_DB_URL = "https://epic7db.com/" + // S3/OSS 配置 S3AccessKey = "s5iWm6wXVvhCNN9nJlXwgWRf" S3SecretKey = "91sTurpFtugXijPg0uSof3JcJma0HED" diff --git a/internal/logic/cron/third_party_sync.go b/internal/logic/cron/third_party_sync.go index 1c027e7..4586a0f 100644 --- a/internal/logic/cron/third_party_sync.go +++ b/internal/logic/cron/third_party_sync.go @@ -51,7 +51,7 @@ func (t *ThirdPartyDataSync) SyncHeroData(ctx context.Context) error { return nil } -// SyncArtifactData 同步神器数据 +// 同步神器数据 func (t *ThirdPartyDataSync) SyncArtifactData(ctx context.Context) error { g.Log().Info(ctx, "开始同步神器数据...") @@ -280,7 +280,7 @@ func (t *ThirdPartyDataSync) processAndSaveHeroData(ctx context.Context, data [] return nil } -// processAndSaveArtifactData 处理并保存神器数据 +// 处理并保存神器数据 func (t *ThirdPartyDataSync) processAndSaveArtifactData(ctx context.Context, data string) error { // 1. 解析json为map var artifactMap map[string]struct { @@ -307,14 +307,21 @@ func (t *ThirdPartyDataSync) processAndSaveArtifactData(ctx context.Context, dat } artifactDbMap := make(map[string]*entity.EpicArtifactInfo, len(dbArtifacts)) for _, a := range dbArtifacts { - artifactDbMap[a.ArtifactCode] = a + artifactDbMap[a.ArtifactNameEn] = a } for _, art := range artifactMap { var dbArt *entity.EpicArtifactInfo - if v, ok := artifactDbMap[art.Code]; ok { + if v, ok := artifactDbMap[art.Name]; ok { dbArt = v } + + // 如果数据库中已有自定义CDN图片，跳过处理 + if dbArt != nil && dbArt.ImageUrl != "" && strings.HasPrefix(dbArt.ImageUrl, consts.S3CustomDomain) { + //g.Log().Debug(ctx, "跳过已有CDN图片的神器:", art.Name) + continue + } + customImgUrl := getArtifactImageUrl(ctx, art.Code, dbArt) artifact := &entity.EpicArtifactInfo{ ArtifactName: i18n.GetZh(ctx, art.Name), @@ -334,8 +341,19 @@ func (t *ThirdPartyDataSync) processAndSaveArtifactData(ctx context.Context, dat artifact.Id = dbArt.Id _, err := dao.EpicArtifactInfo.Ctx(ctx). Where(dao.EpicArtifactInfo.Columns().ArtifactCode, art.Code). - Data(artifact). - Update() + Data(g.Map{ + dao.EpicArtifactInfo.Columns().ArtifactName: artifact.ArtifactName, + dao.EpicArtifactInfo.Columns().ArtifactNameEn: artifact.ArtifactNameEn, + dao.EpicArtifactInfo.Columns().Rarity: artifact.Rarity, + dao.EpicArtifactInfo.Columns().Role: artifact.Role, + dao.EpicArtifactInfo.Columns().StatsAttack: artifact.StatsAttack, + dao.EpicArtifactInfo.Columns().StatsHealth: artifact.StatsHealth, + dao.EpicArtifactInfo.Columns().StatsDefense: artifact.StatsDefense, + dao.EpicArtifactInfo.Columns().ImageUrl: artifact.ImageUrl, + dao.EpicArtifactInfo.Columns().Updater: artifact.Updater, + dao.EpicArtifactInfo.Columns().UpdateTime: gtime.Now(), + dao.EpicArtifactInfo.Columns().Deleted: artifact.Deleted, + }).Update() if err != nil { g.Log().Error(ctx, "更新神器失败:", art.Name, err) continue @@ -343,8 +361,22 @@ func (t *ThirdPartyDataSync) processAndSaveArtifactData(ctx context.Context, dat g.Log().Info(ctx, "更新神器:", art.Name) } else { artifact.Creator = "sync" - artifact.CreateTime = gtime.Now() - _, err := dao.EpicArtifactInfo.Ctx(ctx).Data(artifact).Insert() + _, err := dao.EpicArtifactInfo.Ctx(ctx).Data(g.Map{ + dao.EpicArtifactInfo.Columns().ArtifactName: artifact.ArtifactName, + dao.EpicArtifactInfo.Columns().ArtifactNameEn: artifact.ArtifactNameEn, + dao.EpicArtifactInfo.Columns().ArtifactCode: artifact.ArtifactCode, + dao.EpicArtifactInfo.Columns().Rarity: artifact.Rarity, + dao.EpicArtifactInfo.Columns().Role: artifact.Role, + dao.EpicArtifactInfo.Columns().StatsAttack: artifact.StatsAttack, + dao.EpicArtifactInfo.Columns().StatsHealth: artifact.StatsHealth, + dao.EpicArtifactInfo.Columns().StatsDefense: artifact.StatsDefense, + dao.EpicArtifactInfo.Columns().ImageUrl: artifact.ImageUrl, + dao.EpicArtifactInfo.Columns().Creator: "sync", + dao.EpicArtifactInfo.Columns().CreateTime: gtime.Now(), + dao.EpicArtifactInfo.Columns().Updater: artifact.Updater, + dao.EpicArtifactInfo.Columns().UpdateTime: artifact.UpdateTime, + dao.EpicArtifactInfo.Columns().Deleted: artifact.Deleted, + }).Insert() if err != nil { g.Log().Error(ctx, "插入神器失败:", art.Name, err) continue @@ -492,7 +524,7 @@ func (t *ThirdPartyDataSync) RefreshHeroSetContentByHeroInfo(ctx context.Context } } -//刷新所有角色配装字段 +// 刷新所有角色配装字段 func (t *ThirdPartyDataSync) RefreshAllHeroSetContent(ctx context.Context) error { g.Log().Info(ctx, "开始批量刷新所有角色配装字段...") @@ -527,19 +559,176 @@ func (t *ThirdPartyDataSync) RefreshAllHeroSetContent(ctx context.Context) error return nil } -// 获取神器图片URL，已上传则复用，否则上传 +// 获取神器图片URL，已上传则复用，否则从epic7db.com匹配并上传到OSS func getArtifactImageUrl(ctx context.Context, artCode string, dbArt *entity.EpicArtifactInfo) string { if dbArt != nil && dbArt.ImageUrl != "" && strings.HasPrefix(dbArt.ImageUrl, consts.S3CustomDomain) { + g.Log().Debug(ctx, "神器已有CDN图片，直接返回:", artCode, dbArt.ImageUrl) return dbArt.ImageUrl } - ossObjectKey := fmt.Sprintf("epic/artifact/images/%s.png", artCode) - ossUrl, err := util.DownloadAndUploadToOSS(ctx, "", ossObjectKey) - if err != nil || ossUrl == "" { + + // 取神器英文名，优先用dbArt.ArtifactNameEn，否则artCode + var artifactNameEn string + if dbArt != nil && dbArt.ArtifactNameEn != "" { + artifactNameEn = dbArt.ArtifactNameEn + } else { + artifactNameEn = artCode + } + g.Log().Debug(ctx, "开始匹配神器图片:", artCode, "英文名:", artifactNameEn) + + // 从Redis获取神器爬虫数据 + redisVal, err := util.RedisCache.Get(ctx, "epic7:artifacts") + if err != nil || redisVal == nil { + g.Log().Error(ctx, "获取Redis神器数据失败:", err) return "" } - prefix := consts.S3Endpoint + "/" + consts.S3Bucket - if strings.HasPrefix(ossUrl, prefix) { - return consts.S3CustomDomain + ossUrl[len(prefix):] + var artifactArr []map[string]interface{} + if err := json.Unmarshal([]byte(redisVal.String()), &artifactArr); err != nil { + g.Log().Error(ctx, "解析Redis神器数据失败:", err) + return "" } - return ossUrl + g.Log().Debug(ctx, "Redis中共有", len(artifactArr), "个神器数据") + + normalize := func(s string) string { + r := []rune{} + for _, c := range s { + if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') { + r = append(r, c) + } + } + return strings.ToLower(string(r)) + } + + // 计算字符串相似度（Levenshtein距离） + similarity := func(s1, s2 string) float64 { + if s1 == s2 { + return 1.0 + } + if len(s1) == 0 || len(s2) == 0 { + return 0.0 + } + + // 计算编辑距离 + d := make([][]int, len(s1)+1) + for i := range d { + d[i] = make([]int, len(s2)+1) + } + + for i := 0; i <= len(s1); i++ { + d[i][0] = i + } + for j := 0; j <= len(s2); j++ { + d[0][j] = j + } + + for i := 1; i <= len(s1); i++ { + for j := 1; j <= len(s2); j++ { + if s1[i-1] == s2[j-1] { + d[i][j] = d[i-1][j-1] + } else { + d[i][j] = min(d[i-1][j]+1, min(d[i][j-1]+1, d[i-1][j-1]+1)) + } + } + } + + maxLen := max(len(s1), len(s2)) + if maxLen == 0 { + return 1.0 + } + return 1.0 - float64(d[len(s1)][len(s2)])/float64(maxLen) + } + + localNorm := normalize(artifactNameEn) + g.Log().Debug(ctx, "本地神器名标准化后:", localNorm) + + var bestMatch map[string]interface{} + var bestSimilarity float64 + var bestMatchName string + + // 先尝试精确匹配 + g.Log().Debug(ctx, "开始精确匹配...") + for _, item := range artifactArr { + archName, ok := item["arch_name"].(string) + if !ok { + continue + } + archNorm := normalize(archName) + if archNorm == localNorm { + bestMatch = item + bestMatchName = archName + g.Log().Debug(ctx, "精确匹配成功:", archName, "->", archNorm) + break + } + } + + // 如果精确匹配失败，尝试模糊匹配 + if bestMatch == nil { + g.Log().Debug(ctx, "精确匹配失败，开始模糊匹配...") + for _, item := range artifactArr { + archName, ok := item["arch_name"].(string) + if !ok { + continue + } + archNorm := normalize(archName) + sim := similarity(localNorm, archNorm) + if sim > bestSimilarity { + bestSimilarity = sim + bestMatch = item + bestMatchName = archName + g.Log().Debug(ctx, "发现更高相似度匹配:", archName, "相似度:", sim) + } + } + + // 只有相似度达到90%才使用模糊匹配结果 + if bestSimilarity < 0.9 { + g.Log().Debug(ctx, "模糊匹配相似度不足90%，最高相似度:", bestSimilarity, "最佳匹配:", bestMatchName) + bestMatch = nil + } else { + g.Log().Debug(ctx, "模糊匹配成功，相似度:", bestSimilarity, "匹配名称:", bestMatchName) + } + } + + if bestMatch != nil { + archSrc, ok := bestMatch["arch_src"].(string) + if !ok || archSrc == "" { + g.Log().Error(ctx, "匹配成功但arch_src为空:", bestMatchName) + return "" + } + g.Log().Debug(ctx, "开始下载并上传图片:", bestMatchName, "图片路径:", archSrc) + + // 拼接第三方图片完整URL + imgUrl := consts.EPIC_DB_URL + strings.TrimPrefix(archSrc, "/") + // 上传到OSS + ossObjectKey := fmt.Sprintf("epic/artifact/images/%s.png", artCode) + ossUrl, err := util.DownloadAndUploadToOSS(ctx, imgUrl, ossObjectKey) + if err != nil || ossUrl == "" { + g.Log().Error(ctx, "下载上传图片失败:", bestMatchName, "错误:", err) + return "" + } + prefix := consts.S3Endpoint + "/" + consts.S3Bucket + if strings.HasPrefix(ossUrl, prefix) { + finalUrl := consts.S3CustomDomain + ossUrl[len(prefix):] + g.Log().Debug(ctx, "图片处理成功:", bestMatchName, "最终URL:", finalUrl) + return finalUrl + } + g.Log().Debug(ctx, "图片处理成功:", bestMatchName, "OSS URL:", ossUrl) + return ossUrl + } + + g.Log().Debug(ctx, "神器匹配失败:", artifactNameEn, "标准化后:", localNorm) + return "" +} + +// 辅助函数 +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func max(a, b int) int { + if a > b { + return a + } + return b } diff --git a/internal/logic/hero/hero.go b/internal/logic/hero/hero.go index f5770a3..a9511f3 100644 --- a/internal/logic/hero/hero.go +++ b/internal/logic/hero/hero.go @@ -420,7 +420,7 @@ func (l *Logic) GetHeroDetailByCode(ctx context.Context, code string) (*v1.HeroD return heroDetailVO, nil } -// ClearHeroCache 清理英雄相关缓存 +// 清理英雄相关缓存 func (l *Logic) ClearHeroCache(ctx context.Context, code string) error { redis := g.Redis() diff --git a/manifest/config/config.yaml b/manifest/config/config.yaml index 80d8506..75bb3be 100644 --- a/manifest/config/config.yaml +++ b/manifest/config/config.yaml @@ -19,5 +19,5 @@ database: redis: default: address: "193.112.151.199:6379" - db: 1 + db: 0 pass: "hu123" \ No newline at end of file