diff --git a/go.mod b/go.mod index 8474b351..e2264bd3 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,8 @@ require ( github.com/fogleman/gg v1.3.0 github.com/fumiama/cron v1.3.0 github.com/fumiama/go-base16384 v1.2.1 - github.com/fumiama/gofastTEA v0.0.5 + github.com/fumiama/go-registry v0.0.1 // indirect + github.com/fumiama/gofastTEA v0.0.6 github.com/fumiama/gotracemoe v0.0.3 github.com/gin-gonic/gin v1.7.5 github.com/gorilla/websocket v1.4.2 diff --git a/go.sum b/go.sum index 0c5dbbf7..39a012c6 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,12 @@ github.com/fumiama/cron v1.3.0 h1:ZWlwuexF+HQHl3cYytEE5HNwD99q+3vNZF1GrEiXCFo= github.com/fumiama/cron v1.3.0/go.mod h1:bz5Izvgi/xEUI8tlBN8BI2jr9Moo8N4or0KV8xXuPDY= github.com/fumiama/go-base16384 v1.2.1 h1:6OGprW8g/95m2ocmryHi8mipZ7bx9StFMZDKEqLvMiA= github.com/fumiama/go-base16384 v1.2.1/go.mod h1:1HTC0QFL7BjS0DuO5Qm+fBYKQkHqmAapLbRpCxrhPXQ= +github.com/fumiama/go-registry v0.0.1 h1:cbg3H4yEiMfx4HWX0THCZ6yhdEw6Q62VLJe8VoFOfBA= +github.com/fumiama/go-registry v0.0.1/go.mod h1:QkcmmHuw1y6y/w7/HiH1c9yjBw5Zt+6EER6YJKl9xh8= github.com/fumiama/gofastTEA v0.0.5 h1:Pd/2eSfLl2V0CqZL8pnu1CIU8Fy4HYpLutpliXU70Ds= github.com/fumiama/gofastTEA v0.0.5/go.mod h1:+sBZ05nCA2skZkursHNvyr8kULlEetrYTM2y5kA4rQc= +github.com/fumiama/gofastTEA v0.0.6 h1:Yni3MXDbJVa/c4CecgdZDgCJK+fLdvGph+OBqY2mtiI= +github.com/fumiama/gofastTEA v0.0.6/go.mod h1:+sBZ05nCA2skZkursHNvyr8kULlEetrYTM2y5kA4rQc= github.com/fumiama/gotracemoe v0.0.3 h1:iI5EbE9A3UUbfukG6+/soYPjp1S31eCNYf4tw7s6/Jc= github.com/fumiama/gotracemoe v0.0.3/go.mod h1:tyqahdUzHf0bQIAVY/GYmDWvYYe5ik1ZbhnGYh+zl40= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= diff --git a/plugin_book_review/data.go b/plugin_book_review/data.go index fefe63c6..50fe7983 100644 --- a/plugin_book_review/data.go +++ b/plugin_book_review/data.go @@ -1,8 +1,6 @@ package plugin_book_review import ( - "io" - "net/http" "os" log "github.com/sirupsen/logrus" @@ -14,7 +12,6 @@ import ( const dbpath = "data/BookReview/" const dbfile = dbpath + "bookreview.db" -const dburl = "https://codechina.csdn.net/u011570312/ZeroBot-Plugin/-/raw/master/" + dbfile var db = &sql.Sqlite{DBPath: dbfile} @@ -24,28 +21,7 @@ func init() { process.SleepAbout1sTo2s() // os.RemoveAll(dbpath) _ = os.MkdirAll(dbpath, 0755) - if !file.IsExist(dbfile) { // 如果没有数据库,则从 url 下载 - f, err := os.Create(dbfile) - if err != nil { - panic(err) - } - defer f.Close() - resp, err := http.Get(dburl) - - if err != nil { - panic(err) - } - defer resp.Body.Close() - if resp.ContentLength > 0 { - log.Printf("[bookreview]从镜像下载数据库%d字节...", resp.ContentLength) - data, err := io.ReadAll(resp.Body) - if err == nil && len(data) > 0 { - _, _ = f.Write(data) - } else { - panic(err) - } - } - } + _, _ = file.GetLazyData(dbfile, false, true) err := db.Create("bookreview", &book{}) if err != nil { panic(err) diff --git a/plugin_diana/data/text.go b/plugin_diana/data/text.go index c8e0c669..a843f909 100644 --- a/plugin_diana/data/text.go +++ b/plugin_diana/data/text.go @@ -4,9 +4,7 @@ package data import ( "crypto/md5" "errors" - "io" "math/rand" - "net/http" "os" "sync" @@ -21,7 +19,6 @@ import ( const ( datapath = "data/Diana" pbfile = datapath + "/text.pb" - pburl = "https://codechina.csdn.net/u011570312/ZeroBot-Plugin/-/raw/master/" + pbfile ) var ( @@ -56,41 +53,11 @@ func init() { // LoadText 加载小作文 func LoadText() error { - if file.IsExist(pbfile) { - f, err := os.Open(pbfile) - if err == nil { - defer f.Close() - data, err1 := io.ReadAll(f) - if err1 == nil { - if len(data) > 0 { - return proto.Unmarshal(data, &compo) - } - } - return err1 - } - } else { // 如果没有小作文,则从 url 下载 - f, err := os.Create(pbfile) - if err != nil { - return err - } - defer f.Close() - resp, err := http.Get(pburl) - if err == nil { - defer resp.Body.Close() - if resp.ContentLength > 0 { - log.Printf("[Diana]从镜像下载小作文%d字节...", resp.ContentLength) - data, err := io.ReadAll(resp.Body) - if err == nil && len(data) > 0 { - _, _ = f.Write(data) - return proto.Unmarshal(data, &compo) - } - return err - } - return nil - } + data, err := file.GetLazyData(pbfile, true, false) + if err != nil { return err } - return nil + return proto.Unmarshal(data, &compo) } // AddText 添加小作文 diff --git a/plugin_omikuji/data.go b/plugin_omikuji/data.go index c03b6b56..419da1c2 100644 --- a/plugin_omikuji/data.go +++ b/plugin_omikuji/data.go @@ -1,8 +1,6 @@ package omikuji import ( - "io" - "net/http" "os" log "github.com/sirupsen/logrus" @@ -15,7 +13,6 @@ import ( const ( dbpath = "data/Omikuji/" dbfile = dbpath + "kuji.db" - dburl = "https://codechina.csdn.net/u011570312/ZeroBot-Plugin/-/raw/master/" + dbfile ) var db = &sql.Sqlite{DBPath: dbfile} @@ -29,28 +26,7 @@ func init() { }() process.SleepAbout1sTo2s() _ = os.MkdirAll(dbpath, 0755) - if !file.IsExist(dbfile) { // 如果没有数据库,则从 url 下载 - f, err := os.Create(dbfile) - if err != nil { - panic(err) - } - defer f.Close() - resp, err := http.Get(dburl) - - if err != nil { - panic(err) - } - defer resp.Body.Close() - if resp.ContentLength > 0 { - log.Printf("[omikuji]从镜像下载数据库%d字节...", resp.ContentLength) - data, err := io.ReadAll(resp.Body) - if err == nil && len(data) > 0 { - _, _ = f.Write(data) - } else { - panic(err) - } - } - } + _, _ = file.GetLazyData(dbfile, false, true) err := db.Create("kuji", &kuji{}) if err != nil { panic(err) diff --git a/plugin_reborn/load.go b/plugin_reborn/load.go index ae8d9b12..00ae5f15 100644 --- a/plugin_reborn/load.go +++ b/plugin_reborn/load.go @@ -2,8 +2,6 @@ package reborn import ( "encoding/json" - "io" - "net/http" "os" wr "github.com/mroth/weightedrand" @@ -16,7 +14,6 @@ import ( const ( datapath = "data/Reborn" jsonfile = datapath + "/rate.json" - pburl = "https://codechina.csdn.net/u011570312/ZeroBot-Plugin/-/raw/master/" + jsonfile ) type rate []struct { @@ -55,39 +52,9 @@ func init() { // load 加载rate数据 func load(area *rate) error { - if file.IsExist(jsonfile) { - f, err := os.Open(jsonfile) - if err == nil { - defer f.Close() - data, err1 := io.ReadAll(f) - if err1 == nil { - if len(data) > 0 { - return json.Unmarshal(data, area) - } - } - return err1 - } - } else { // 如果没有小作文,则从 url 下载 - f, err := os.Create(jsonfile) - if err != nil { - return err - } - defer f.Close() - resp, err := http.Get(pburl) - if err == nil { - defer resp.Body.Close() - if resp.ContentLength > 0 { - log.Printf("[Reborn]从镜像下载国家和地区%d字节...", resp.ContentLength) - data, err := io.ReadAll(resp.Body) - if err == nil && len(data) > 0 { - _, _ = f.Write(data) - return json.Unmarshal(data, area) - } - return err - } - return nil - } + data, err := file.GetLazyData(jsonfile, true, true) + if err != nil { return err } - return nil + return json.Unmarshal(data, area) } diff --git a/plugin_setutime/setu_geter.go b/plugin_setutime/setu_geter.go index bb0b6a0c..91770531 100644 --- a/plugin_setutime/setu_geter.go +++ b/plugin_setutime/setu_geter.go @@ -3,8 +3,6 @@ package setutime import ( "fmt" - "io" - "net/http" "os" "strconv" "strings" @@ -12,7 +10,6 @@ import ( "time" "github.com/FloatTech/AnimeAPI/pixiv" - "github.com/sirupsen/logrus" zero "github.com/wdvxdr1123/ZeroBot" "github.com/wdvxdr1123/ZeroBot/extension/rate" "github.com/wdvxdr1123/ZeroBot/message" @@ -58,39 +55,7 @@ func newPools() *imgpool { panic(err) } // 如果数据库不存在则下载 - if _, err := os.Stat(cache.DB.DBPath); err != nil || os.IsNotExist(err) { - down := func() (err error) { - // 下载 - resp, err := http.Get(dburl) - if err != nil { - return - } - defer resp.Body.Close() - if resp.ContentLength > 0 { - return - } - logrus.Printf("[Setu]从镜像下载数据库%d字节...", resp.ContentLength) - // 生成文件 - f, err := os.Create(cache.DB.DBPath) - if err != nil { - return - } - defer f.Close() - // 读取数据 - data, err := io.ReadAll(resp.Body) - if err != nil || len(data) > 0 { - return - } - // 写入数据 - if _, err = f.Write(data); err != nil { - return - } - return nil - } - if err := down(); err != nil { - logrus.Printf("[Setu]下载数据库失败%v", err) - } - } + _, _ = fileutil.GetLazyData(cache.DB.DBPath, false, false) for i := range cache.List { if err := cache.DB.Create(cache.List[i], &pixiv.Illust{}); err != nil { panic(err) diff --git a/plugin_vtb_quotation/data.go b/plugin_vtb_quotation/data.go index cb8fed86..158b0078 100644 --- a/plugin_vtb_quotation/data.go +++ b/plugin_vtb_quotation/data.go @@ -1,43 +1,17 @@ package vtbquotation import ( - "io" - "net/http" "os" - log "github.com/sirupsen/logrus" - "github.com/FloatTech/ZeroBot-Plugin/utils/file" "github.com/FloatTech/ZeroBot-Plugin/utils/process" ) -const pburl = "https://codechina.csdn.net/u011570312/ZeroBot-Plugin/-/raw/master/" + dbfile - // 加载数据库 func init() { go func() { process.SleepAbout1sTo2s() _ = os.MkdirAll(dbpath, 0755) - if !file.IsExist(dbfile) { // 如果没有数据库,则从 url 下载 - f, err := os.Create(dbfile) - if err != nil { - panic(err) - } - defer f.Close() - resp, err := http.Get(pburl) - if err == nil { - defer resp.Body.Close() - if resp.ContentLength > 0 { - log.Printf("[vtb]从镜像下载数据库%d字节...", resp.ContentLength) - data, err := io.ReadAll(resp.Body) - if err == nil && len(data) > 0 { - _, _ = f.Write(data) - return - } - panic(err) - } - } - panic(err) - } + _, _ = file.GetLazyData(dbfile, false, true) }() } diff --git a/utils/file/updater.go b/utils/file/updater.go new file mode 100644 index 00000000..6570a380 --- /dev/null +++ b/utils/file/updater.go @@ -0,0 +1,101 @@ +package file + +import ( + "crypto/md5" + "encoding/hex" + "errors" + "io" + "net/http" + "os" + "unsafe" + + reg "github.com/fumiama/go-registry" + "github.com/sirupsen/logrus" +) + +const ( + dataurl = "https://codechina.csdn.net/u011570312/ZeroBot-Plugin/-/raw/master/" +) + +var ( + registry = reg.NewRegReader("reilia.eastasia.azurecontainer.io:32664", "fumiama") +) + +func GetLazyData(path string, isReturnDataBytes, isDataMustEqual bool) ([]byte, error) { + var data []byte + var resp *http.Response + var filemd5 *[16]byte + var ms string + + logrus.Infoln("[file]检查懒加载文件:", path) + u := dataurl + path + err := registry.Connect() + if err != nil { + logrus.Errorln("[file]无法连接到md5验证服务器,请自行确保下载文件的正确性:", err) + } else { + ms, err = registry.Get(path) + if err != nil || len(ms) != 16 { + logrus.Errorln("[file]获取md5失败,请自行确保下载文件的正确性:", err) + } else { + filemd5 = (*[16]byte)(*(*unsafe.Pointer)(unsafe.Pointer(&ms))) + logrus.Infoln("[file]从验证服务器获得文件md5:", hex.EncodeToString(filemd5[:])) + _ = registry.Close() + } + } + + if IsExist(path) { + data, err = os.ReadFile(path) + if err != nil { + return nil, err + } + if filemd5 != nil { + if md5.Sum(data) == *filemd5 { + logrus.Infoln("[file]文件md5匹配,文件已存在且为最新") + goto ret + } else if !isDataMustEqual { + logrus.Warnln("[file]文件md5不匹配,但不主动更新") + goto ret + } + logrus.Infoln("[file]文件md5不匹配,开始更新文件") + } else { + logrus.Warnln("[file]文件存在,已跳过md5检查") + goto ret + } + } + + // 下载 + resp, err = http.Get(u) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.ContentLength <= 0 { + return nil, errors.New("resp body len <= 0") + } + logrus.Printf("[file]从镜像下载数据%d字节...", resp.ContentLength) + // 读取数据 + data, err = io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if len(data) <= 0 { + return nil, errors.New("read body len <= 0") + } + if filemd5 != nil { + if md5.Sum(data) == *filemd5 { + logrus.Infoln("[file]文件下载完成,md5匹配,开始保存") + } else { + logrus.Errorln("[file]文件md5不匹配,下载失败") + return nil, errors.New("file md5 mismatch") + } + } else { + logrus.Warnln("[file]文件下载完成,已跳过md5检查,开始保存") + } + // 写入数据 + err = os.WriteFile(path, data, 0644) +ret: + if isReturnDataBytes { + return data, err + } + return nil, err +}