package cron import ( "context" "encoding/json" "epic/internal/consts" "epic/internal/dao" "epic/internal/logic/i18n" "epic/internal/model/dto" "epic/internal/model/entity" "epic/internal/util" "fmt" "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/gclient" "github.com/gogf/gf/v2/os/gtime" "strconv" "strings" "time" ) // ThirdPartyDataSync 第三方数据同步器 type ThirdPartyDataSync struct { client *gclient.Client } // NewThirdPartyDataSync 创建第三方数据同步器 func NewThirdPartyDataSync() *ThirdPartyDataSync { return &ThirdPartyDataSync{ client: gclient.New().Timeout(30 * time.Second), } } func (t *ThirdPartyDataSync) SyncHeroData(ctx context.Context) error { g.Log().Info(ctx, "开始同步英雄数据...") // 示例:从第三方API获取英雄数据 heroData, err := t.fetchHeroDataFromAPI(ctx) if err != nil || heroData == nil { g.Log().Error(ctx, "获取英雄数据失败:", err) return err } // 处理并保存数据 if err := t.processAndSaveHeroData(ctx, heroData); err != nil { g.Log().Error(ctx, "处理英雄数据失败:", err) return err } g.Log().Info(ctx, "英雄数据同步完成") return nil } // 同步神器数据 func (t *ThirdPartyDataSync) SyncArtifactData(ctx context.Context) error { g.Log().Info(ctx, "开始同步神器数据...") //从第三方API获取神器数据 artifactData, err := t.fetchArtifactDataFromAPI(ctx) if err != nil { g.Log().Error(ctx, "获取神器数据失败:", err) return err } // 处理并保存数据 if err := t.processAndSaveArtifactData(ctx, artifactData); err != nil { g.Log().Error(ctx, "处理神器数据失败:", err) return err } g.Log().Info(ctx, "神器数据同步完成") return nil } // fetchHeroDataFromAPI 从API获取英雄数据 func (t *ThirdPartyDataSync) fetchHeroDataFromAPI(ctx context.Context) ([]byte, error) { // 示例API地址,实际使用时需要替换为真实的API apiURL := consts.HeroListURL // 添加请求头 headers := map[string]string{ "User-Agent": "EpicGameBot/1.0", "Accept": "application/json", } // 发送GET请求 resp, err := t.client.Header(headers).Get(ctx, apiURL) if err != nil { return nil, fmt.Errorf("API请求失败: %v", err) } defer resp.Close() // 检查响应状态 if resp.StatusCode != 200 { return nil, fmt.Errorf("API响应错误,状态码: %d", resp.StatusCode) } // 读取响应内容 content := resp.ReadAll() g.Log().Debug(ctx, "API响应内容长度:", len(content)) return content, nil } // fetchHeroBuildsFromAPI 通过角色英文名POST请求获取配装数据 func (t *ThirdPartyDataSync) FetchHeroBuildsFromAPI(ctx context.Context, heroName string) (string, error) { apiURL := consts.HeroNameURL headers := map[string]string{ "User-Agent": "EpicGameBot/1.0", "Accept": "application/json", "Content-Type": "application/json", } heroNameEN := i18n.Zh2En(heroName) resp, err := t.client.Header(headers).Post(ctx, apiURL, heroNameEN) if err != nil { return "", fmt.Errorf("API请求失败: %v", err) } defer resp.Close() if resp.StatusCode != 200 { return "", fmt.Errorf("API响应错误,状态码: %d", resp.StatusCode) } content := resp.ReadAll() g.Log().Debug(ctx, "配装API响应内容长度:", len(content)) return string(content), nil } // 从API获取神器数据 func (t *ThirdPartyDataSync) fetchArtifactDataFromAPI(ctx context.Context) (string, error) { // 示例API地址 apiURL := consts.ArtifactDataURL headers := map[string]string{ "Accept": "application/json", } resp, err := t.client.Header(headers).Get(ctx, apiURL) if err != nil { return "", fmt.Errorf("API请求失败: %v", err) } defer resp.Close() if resp.StatusCode != 200 { return "", fmt.Errorf("API响应错误,状态码: %d", resp.StatusCode) } content := resp.ReadAll() g.Log().Debug(ctx, "神器API响应内容长度:", len(content)) return string(content), nil } func (t *ThirdPartyDataSync) processAndSaveHeroData(ctx context.Context, data []byte) error { // 使用 gjson 解析 j := gjson.New(data) if j == nil || j.IsNil() { return fmt.Errorf("英雄数据格式错误,期望是一个JSON对象") } // 先解析为 map[string]*ThirdPartyHeroDTO var heroes map[string]*dto.ThirdPartyHeroDTO if err := j.Scan(&heroes); err != nil { return fmt.Errorf("解析英雄数据到DTO失败: %v", err) } g.Log().Info(ctx, "解析到", len(heroes), "个英雄数据") // 一次性查出所有数据库英雄,构建map var dbHeroes []*entity.EpicHeroInfo err := dao.EpicHeroInfo.Ctx(ctx).Scan(&dbHeroes) if err != nil { return fmt.Errorf("查询数据库英雄失败: %v", err) } heroMap := make(map[string]*entity.EpicHeroInfo, len(dbHeroes)) for _, h := range dbHeroes { heroMap[h.HeroCode] = h } // 遍历 map,设置 Name 字段,并保存 for name, hero := range heroes { hero.Name = name // 将 map 的 key 作为 Name 字段 dbHero, exists := heroMap[hero.Code] if exists && dbHero.RawJson != "" { // 已有且rawJson有值,跳过 continue } if exists && dbHero.RawJson == "" { // 只更新 rawJson 字段 rawJsonBytes, err := json.Marshal(hero) if err != nil { g.Log().Error(ctx, "序列化英雄数据失败:", err) continue } rawJson := string(rawJsonBytes) _, err = dao.EpicHeroInfo.Ctx(ctx). Where(dao.EpicHeroInfo.Columns().HeroCode, hero.Code). Data(g.Map{ dao.EpicHeroInfo.Columns().RawJson: rawJson, dao.EpicHeroInfo.Columns().UpdateTime: gtime.Now(), }). Update() if err != nil { g.Log().Error(ctx, "更新英雄rawJson失败:", err) continue } g.Log().Debug(ctx, "更新英雄rawJson:", hero.Code) continue } // 新增逻辑保持原样 status60 := hero.CalculatedStatus.Lv60SixStarFullyAwakened status60json, _ := gjson.EncodeString(status60) heroJson, _ := gjson.EncodeString(hero) zhHeroName := i18n.GetZh(ctx, hero.Name) //zhRole := i18n.GetZh(ctx, hero.Role) //zhAttribute := i18n.GetZh(ctx, hero.Attribute) fmt.Println(hero.Assets.Image) // 上传图片到 OSS,并获取自定义域名的访问路径 ossObjectKey := fmt.Sprintf("epic/hero/images/%s.png", hero.Code) ossUrl, err := util.DownloadAndUploadToOSS(ctx, hero.Assets.Icon, ossObjectKey) if err != nil { g.Log().Error(ctx, "上传英雄图片到OSS失败:", err) return err // 直接返回,后续数据库不会插入 } fmt.Println(ossUrl) // 替换为自定义域名 customImgUrl := "" if ossUrl != "" { prefix := consts.S3Endpoint + "/" + consts.S3Bucket if len(ossUrl) > len(prefix) && ossUrl[:len(prefix)] == prefix { customImgUrl = consts.S3CustomDomain + ossUrl[len(prefix):] } else { customImgUrl = ossUrl // fallback } } newHero := &entity.EpicHeroInfo{ Id: 0, HeroName: zhHeroName, HeroCode: hero.Code, HeroAttrLv60: status60json, Creator: "", CreateTime: gtime.Now(), Updater: "", UpdateTime: gtime.Now(), Deleted: false, //NickName: "", Rarity: strconv.Itoa(hero.Rarity), Role: hero.Role, //Zodiac: "", HeadImgUrl: customImgUrl, Attribute: hero.Attribute, Remark: "", RawJson: heroJson, } _, err = dao.EpicHeroInfo.Ctx(ctx).Data(newHero).Insert() if err != nil { g.Log().Error(ctx, "插入新英雄失败:", err) continue } g.Log().Debug(ctx, "插入新英雄:", hero.Code) // 新增后立即刷新该图片的OSS预签名URL到缓存 if customImgUrl != "" { key := ossObjectKey err := util.RefreshOssPresignedUrlCache(ctx, []string{key}, 24*time.Hour) if err != nil { g.Log().Error(ctx, "刷新新英雄图片预签名URL失败:", key, err) } } } return nil } // 处理并保存神器数据 func (t *ThirdPartyDataSync) processAndSaveArtifactData(ctx context.Context, data string) error { // 1. 解析json为map var artifactMap map[string]struct { Name string `json:"name"` Rarity int `json:"rarity"` Role string `json:"role"` Stats struct { Attack int `json:"attack"` Health int `json:"health"` Defense int `json:"defense"` } `json:"stats"` Code string `json:"code"` } if err := json.Unmarshal([]byte(data), &artifactMap); err != nil { return fmt.Errorf("解析神器数据失败: %v", err) } g.Log().Info(ctx, "解析到", len(artifactMap), "个神器数据") // 2. 查询数据库所有神器,构建map var dbArtifacts []*entity.EpicArtifactInfo err := dao.EpicArtifactInfo.Ctx(ctx).Scan(&dbArtifacts) if err != nil { return fmt.Errorf("查询数据库神器失败: %v", err) } artifactDbMap := make(map[string]*entity.EpicArtifactInfo, len(dbArtifacts)) for _, a := range dbArtifacts { artifactDbMap[a.ArtifactNameEn] = a } for _, art := range artifactMap { var dbArt *entity.EpicArtifactInfo if v, ok := artifactDbMap[art.Name]; ok { dbArt = v } // 如果数据库中已有自定义CDN图片,跳过处理 if dbArt != nil && dbArt.ImageUrl != "" && strings.HasPrefix(dbArt.ImageUrl, consts.S3CustomDomain) { //g.Log().Debug(ctx, "跳过已有CDN图片的神器:", art.Name) continue } customImgUrl := getArtifactImageUrl(ctx, art.Code, dbArt) artifact := &entity.EpicArtifactInfo{ ArtifactName: i18n.GetZh(ctx, art.Name), ArtifactNameEn: art.Name, ArtifactCode: art.Code, Rarity: strconv.Itoa(art.Rarity), Role: art.Role, StatsAttack: art.Stats.Attack, StatsHealth: art.Stats.Health, StatsDefense: art.Stats.Defense, ImageUrl: customImgUrl, Updater: "sync", UpdateTime: gtime.Now(), Deleted: false, } if dbArt != nil { artifact.Id = dbArt.Id _, err := dao.EpicArtifactInfo.Ctx(ctx). Where(dao.EpicArtifactInfo.Columns().ArtifactCode, art.Code). Data(g.Map{ dao.EpicArtifactInfo.Columns().ArtifactName: artifact.ArtifactName, dao.EpicArtifactInfo.Columns().ArtifactNameEn: artifact.ArtifactNameEn, dao.EpicArtifactInfo.Columns().Rarity: artifact.Rarity, dao.EpicArtifactInfo.Columns().Role: artifact.Role, dao.EpicArtifactInfo.Columns().StatsAttack: artifact.StatsAttack, dao.EpicArtifactInfo.Columns().StatsHealth: artifact.StatsHealth, dao.EpicArtifactInfo.Columns().StatsDefense: artifact.StatsDefense, dao.EpicArtifactInfo.Columns().ImageUrl: artifact.ImageUrl, dao.EpicArtifactInfo.Columns().Updater: artifact.Updater, dao.EpicArtifactInfo.Columns().UpdateTime: gtime.Now(), dao.EpicArtifactInfo.Columns().Deleted: artifact.Deleted, }).Update() if err != nil { g.Log().Error(ctx, "更新神器失败:", art.Name, err) continue } g.Log().Info(ctx, "更新神器:", art.Name) } else { artifact.Creator = "sync" _, err := dao.EpicArtifactInfo.Ctx(ctx).Data(g.Map{ dao.EpicArtifactInfo.Columns().ArtifactName: artifact.ArtifactName, dao.EpicArtifactInfo.Columns().ArtifactNameEn: artifact.ArtifactNameEn, dao.EpicArtifactInfo.Columns().ArtifactCode: artifact.ArtifactCode, dao.EpicArtifactInfo.Columns().Rarity: artifact.Rarity, dao.EpicArtifactInfo.Columns().Role: artifact.Role, dao.EpicArtifactInfo.Columns().StatsAttack: artifact.StatsAttack, dao.EpicArtifactInfo.Columns().StatsHealth: artifact.StatsHealth, dao.EpicArtifactInfo.Columns().StatsDefense: artifact.StatsDefense, dao.EpicArtifactInfo.Columns().ImageUrl: artifact.ImageUrl, dao.EpicArtifactInfo.Columns().Creator: "sync", dao.EpicArtifactInfo.Columns().CreateTime: gtime.Now(), dao.EpicArtifactInfo.Columns().Updater: artifact.Updater, dao.EpicArtifactInfo.Columns().UpdateTime: artifact.UpdateTime, dao.EpicArtifactInfo.Columns().Deleted: artifact.Deleted, }).Insert() if err != nil { g.Log().Error(ctx, "插入神器失败:", art.Name, err) continue } g.Log().Info(ctx, "插入神器:", art.Name) } } return nil } // saveHeroData 保存单个英雄数据 func (t *ThirdPartyDataSync) saveHeroData(ctx context.Context, hero *dto.ThirdPartyHeroDTO) error { // 查询是否存在 var dbHero *entity.EpicHeroInfo err := dao.EpicHeroInfo.Ctx(ctx). Where(dao.EpicHeroInfo.Columns().HeroCode, hero.Code). Scan(&dbHero) if err != nil { return err } // 获取原始 JSON 字符串 rawJsonBytes, err := json.Marshal(hero) if err != nil { return err } rawJson := string(rawJsonBytes) if dbHero != nil && dbHero.HeroCode != "" { // 查到记录 if dbHero.RawJson == "" { // 只更新 rawJson 字段 _, err = dao.EpicHeroInfo.Ctx(ctx). Where(dao.EpicHeroInfo.Columns().HeroCode, hero.Code). Data(g.Map{ dao.EpicHeroInfo.Columns().RawJson: rawJson, dao.EpicHeroInfo.Columns().UpdateTime: gtime.Now(), }). Update() if err != nil { return err } g.Log().Debug(ctx, "更新英雄rawJson:", hero.Code) } // 已有 rawJson,不做处理 } else { status60 := hero.CalculatedStatus.Lv60SixStarFullyAwakened status60json, _ := gjson.EncodeString(status60) heroJson, _ := gjson.EncodeString(hero) // 使用i18n服务转换字段 zhHeroName := i18n.GetZh(ctx, hero.Name) //zhRole := i18n.GetZh(ctx, hero.Role) //zhAttribute := i18n.GetZh(ctx, hero.Attribute) newHero := &entity.EpicHeroInfo{ Id: 0, HeroName: zhHeroName, HeroCode: hero.Code, HeroAttrLv60: status60json, Creator: "", CreateTime: gtime.Now(), Updater: "", UpdateTime: gtime.Now(), Deleted: false, //NickName: nil, Rarity: strconv.Itoa(hero.Rarity), Role: hero.Role, Zodiac: "", HeadImgUrl: "", Attribute: hero.Attribute, Remark: "", RawJson: heroJson, } // 没查到,插入新记录,字段按 DO 结构体补全 _, err = dao.EpicHeroInfo.Ctx(ctx).Data(newHero).Insert() if err != nil { return err } g.Log().Debug(ctx, "插入新英雄:", hero.Code) } return nil } // 保存单个神器数据 func (t *ThirdPartyDataSync) saveArtifactData(ctx context.Context, artifact *dto.ThirdPartyArtifactDTO) error { // TODO: 实现具体的数据库保存逻辑 // 现在 artifact 是一个强类型对象, 可以直接使用 artifact.Code, artifact.Name 等 // 示例:记录同步日志 syncLog := map[string]interface{}{ "type": "artifact_sync", "artifact_code": artifact.Code, "sync_time": gtime.Now(), "status": "success", } g.Log().Debug(ctx, "保存神器数据:", syncLog) return nil } // SyncAllData 同步所有数据 func (t *ThirdPartyDataSync) SyncAllData(ctx context.Context) error { g.Log().Info(ctx, "开始同步所有第三方数据...") // 同步英雄数据 if err := t.SyncHeroData(ctx); err != nil { g.Log().Error(ctx, "英雄数据同步失败:", err) // 继续同步其他数据 } // 同步神器数据 if err := t.SyncArtifactData(ctx); err != nil { g.Log().Error(ctx, "神器数据同步失败:", err) // 继续同步其他数据 } // 可以继续添加其他数据类型的同步 g.Log().Info(ctx, "所有第三方数据同步完成") return nil } // RefreshHeroSetContentByHeroInfo 刷新单个角色配装字段 func (t *ThirdPartyDataSync) RefreshHeroSetContentByHeroInfo(ctx context.Context, heroName, heroCode, jsonStr string) error { g.Log().Infof(ctx, "刷新角色配装: %s", heroName) if len(jsonStr) > 200 { _, err := dao.EpicHeroInfo.Ctx(ctx). Where(dao.EpicHeroInfo.Columns().HeroCode, heroCode). Data(g.Map{ dao.EpicHeroInfo.Columns().SetContentJson: jsonStr, dao.EpicHeroInfo.Columns().SetUpdateTime: gtime.Now(), }). Update() if err != nil { g.Log().Errorf(ctx, "更新数据库失败: %s, err: %v", heroName, err) return err } g.Log().Infof(ctx, "已更新: %s", heroName) return nil } else { g.Log().Error(ctx, "配装数据无效(长度<=200): %s", heroName) return fmt.Errorf("配装数据无效(长度<=200)") } } // 刷新所有角色配装字段 func (t *ThirdPartyDataSync) RefreshAllHeroSetContent(ctx context.Context) error { g.Log().Info(ctx, "开始批量刷新所有角色配装字段...") // 1. 查询所有角色,按set_update_time正序排列,为空的在最前面 var heroList []*entity.EpicHeroInfo err := dao.EpicHeroInfo.Ctx(ctx). OrderAsc("set_update_time"). Scan(&heroList) if err != nil { g.Log().Error(ctx, "查询epic_hero_info失败:", err) return err } for i, epicHeroInfo := range heroList { g.Log().Infof(ctx, "[%d/%d] 处理角色: %s", i+1, len(heroList), epicHeroInfo.HeroName) jsonStr, err := t.FetchHeroBuildsFromAPI(ctx, epicHeroInfo.HeroName) if err != nil { g.Log().Errorf(ctx, "获取配装数据失败: %s, err: %v", epicHeroInfo.HeroName, err) } else { err = t.RefreshHeroSetContentByHeroInfo(ctx, epicHeroInfo.HeroName, epicHeroInfo.HeroCode, jsonStr) } // 远程接口有频率限制,停顿10分钟 if i < len(heroList)-1 { g.Log().Info(ctx, "等待10分钟...") time.Sleep(10 * time.Minute) } if err != nil { continue } } g.Log().Info(ctx, "所有角色配装字段刷新完成") return nil } // 获取神器图片URL,已上传则复用,否则从epic7db.com匹配并上传到OSS func getArtifactImageUrl(ctx context.Context, artCode string, dbArt *entity.EpicArtifactInfo) string { if dbArt != nil && dbArt.ImageUrl != "" && strings.HasPrefix(dbArt.ImageUrl, consts.S3CustomDomain) { g.Log().Debug(ctx, "神器已有CDN图片,直接返回:", artCode, dbArt.ImageUrl) return dbArt.ImageUrl } // 取神器英文名,优先用dbArt.ArtifactNameEn,否则artCode var artifactNameEn string if dbArt != nil && dbArt.ArtifactNameEn != "" { artifactNameEn = dbArt.ArtifactNameEn } else { artifactNameEn = artCode } g.Log().Debug(ctx, "开始匹配神器图片:", artCode, "英文名:", artifactNameEn) // 从Redis获取神器爬虫数据 redisVal, err := util.RedisCache.Get(ctx, "epic7:artifacts") if err != nil || redisVal == nil { g.Log().Error(ctx, "获取Redis神器数据失败:", err) return "" } var artifactArr []map[string]interface{} if err := json.Unmarshal([]byte(redisVal.String()), &artifactArr); err != nil { g.Log().Error(ctx, "解析Redis神器数据失败:", err) return "" } g.Log().Debug(ctx, "Redis中共有", len(artifactArr), "个神器数据") normalize := func(s string) string { r := []rune{} for _, c := range s { if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') { r = append(r, c) } } return strings.ToLower(string(r)) } // 计算字符串相似度(Levenshtein距离) similarity := func(s1, s2 string) float64 { if s1 == s2 { return 1.0 } if len(s1) == 0 || len(s2) == 0 { return 0.0 } // 计算编辑距离 d := make([][]int, len(s1)+1) for i := range d { d[i] = make([]int, len(s2)+1) } for i := 0; i <= len(s1); i++ { d[i][0] = i } for j := 0; j <= len(s2); j++ { d[0][j] = j } for i := 1; i <= len(s1); i++ { for j := 1; j <= len(s2); j++ { if s1[i-1] == s2[j-1] { d[i][j] = d[i-1][j-1] } else { d[i][j] = min(d[i-1][j]+1, min(d[i][j-1]+1, d[i-1][j-1]+1)) } } } maxLen := max(len(s1), len(s2)) if maxLen == 0 { return 1.0 } return 1.0 - float64(d[len(s1)][len(s2)])/float64(maxLen) } localNorm := normalize(artifactNameEn) g.Log().Debug(ctx, "本地神器名标准化后:", localNorm) var bestMatch map[string]interface{} var bestSimilarity float64 var bestMatchName string // 先尝试精确匹配 g.Log().Debug(ctx, "开始精确匹配...") for _, item := range artifactArr { archName, ok := item["arch_name"].(string) if !ok { continue } archNorm := normalize(archName) if archNorm == localNorm { bestMatch = item bestMatchName = archName g.Log().Debug(ctx, "精确匹配成功:", archName, "->", archNorm) break } } // 如果精确匹配失败,尝试模糊匹配 if bestMatch == nil { g.Log().Debug(ctx, "精确匹配失败,开始模糊匹配...") for _, item := range artifactArr { archName, ok := item["arch_name"].(string) if !ok { continue } archNorm := normalize(archName) sim := similarity(localNorm, archNorm) if sim > bestSimilarity { bestSimilarity = sim bestMatch = item bestMatchName = archName g.Log().Debug(ctx, "发现更高相似度匹配:", archName, "相似度:", sim) } } // 只有相似度达到90%才使用模糊匹配结果 if bestSimilarity < 0.9 { g.Log().Debug(ctx, "模糊匹配相似度不足90%,最高相似度:", bestSimilarity, "最佳匹配:", bestMatchName) bestMatch = nil } else { g.Log().Debug(ctx, "模糊匹配成功,相似度:", bestSimilarity, "匹配名称:", bestMatchName) } } if bestMatch != nil { archSrc, ok := bestMatch["arch_src"].(string) if !ok || archSrc == "" { g.Log().Error(ctx, "匹配成功但arch_src为空:", bestMatchName) return "" } g.Log().Debug(ctx, "开始下载并上传图片:", bestMatchName, "图片路径:", archSrc) // 拼接第三方图片完整URL imgUrl := consts.EPIC_DB_URL + strings.TrimPrefix(archSrc, "/") // 上传到OSS ossObjectKey := fmt.Sprintf("epic/artifact/images/%s.png", artCode) ossUrl, err := util.DownloadAndUploadToOSS(ctx, imgUrl, ossObjectKey) if err != nil || ossUrl == "" { g.Log().Error(ctx, "下载上传图片失败:", bestMatchName, "错误:", err) return "" } prefix := consts.S3Endpoint + "/" + consts.S3Bucket if strings.HasPrefix(ossUrl, prefix) { finalUrl := consts.S3CustomDomain + ossUrl[len(prefix):] g.Log().Debug(ctx, "图片处理成功:", bestMatchName, "最终URL:", finalUrl) return finalUrl } g.Log().Debug(ctx, "图片处理成功:", bestMatchName, "OSS URL:", ossUrl) return ossUrl } g.Log().Debug(ctx, "神器匹配失败:", artifactNameEn, "标准化后:", localNorm) return "" } // 辅助函数 func min(a, b int) int { if a < b { return a } return b } func max(a, b int) int { if a > b { return a } return b }