✏️ 统一懒加载验证

go.mod

@@ -11,7 +11,8 @@ require (
 	github.com/fogleman/gg v1.3.0
 	github.com/fumiama/cron v1.3.0
 	github.com/fumiama/go-base16384 v1.2.1
-	github.com/fumiama/gofastTEA v0.0.5
+	github.com/fumiama/go-registry v0.0.1 // indirect
+	github.com/fumiama/gofastTEA v0.0.6
 	github.com/fumiama/gotracemoe v0.0.3
 	github.com/gin-gonic/gin v1.7.5
 	github.com/gorilla/websocket v1.4.2

go.sum

@@ -43,8 +43,12 @@ github.com/fumiama/cron v1.3.0 h1:ZWlwuexF+HQHl3cYytEE5HNwD99q+3vNZF1GrEiXCFo=
 github.com/fumiama/cron v1.3.0/go.mod h1:bz5Izvgi/xEUI8tlBN8BI2jr9Moo8N4or0KV8xXuPDY=
 github.com/fumiama/go-base16384 v1.2.1 h1:6OGprW8g/95m2ocmryHi8mipZ7bx9StFMZDKEqLvMiA=
 github.com/fumiama/go-base16384 v1.2.1/go.mod h1:1HTC0QFL7BjS0DuO5Qm+fBYKQkHqmAapLbRpCxrhPXQ=
+github.com/fumiama/go-registry v0.0.1 h1:cbg3H4yEiMfx4HWX0THCZ6yhdEw6Q62VLJe8VoFOfBA=
+github.com/fumiama/go-registry v0.0.1/go.mod h1:QkcmmHuw1y6y/w7/HiH1c9yjBw5Zt+6EER6YJKl9xh8=
 github.com/fumiama/gofastTEA v0.0.5 h1:Pd/2eSfLl2V0CqZL8pnu1CIU8Fy4HYpLutpliXU70Ds=
 github.com/fumiama/gofastTEA v0.0.5/go.mod h1:+sBZ05nCA2skZkursHNvyr8kULlEetrYTM2y5kA4rQc=
+github.com/fumiama/gofastTEA v0.0.6 h1:Yni3MXDbJVa/c4CecgdZDgCJK+fLdvGph+OBqY2mtiI=
+github.com/fumiama/gofastTEA v0.0.6/go.mod h1:+sBZ05nCA2skZkursHNvyr8kULlEetrYTM2y5kA4rQc=
 github.com/fumiama/gotracemoe v0.0.3 h1:iI5EbE9A3UUbfukG6+/soYPjp1S31eCNYf4tw7s6/Jc=
 github.com/fumiama/gotracemoe v0.0.3/go.mod h1:tyqahdUzHf0bQIAVY/GYmDWvYYe5ik1ZbhnGYh+zl40=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=

plugin_book_review/data.go

@@ -1,8 +1,6 @@
 package plugin_book_review
 
 import (
-	"io"
-	"net/http"
 	"os"
 
 	log "github.com/sirupsen/logrus"
@@ -14,7 +12,6 @@ import (
 
 const dbpath = "data/BookReview/"
 const dbfile = dbpath + "bookreview.db"
-const dburl = "https://codechina.csdn.net/u011570312/ZeroBot-Plugin/-/raw/master/" + dbfile
 
 var db = &sql.Sqlite{DBPath: dbfile}
 
@@ -24,28 +21,7 @@ func init() {
 		process.SleepAbout1sTo2s()
 		// os.RemoveAll(dbpath)
 		_ = os.MkdirAll(dbpath, 0755)
-		if !file.IsExist(dbfile) { // 如果没有数据库，则从 url 下载
+		_, _ = file.GetLazyData(dbfile, false, true)
-			f, err := os.Create(dbfile)
-			if err != nil {
-				panic(err)
-			}
-			defer f.Close()
-			resp, err := http.Get(dburl)
-
-			if err != nil {
-				panic(err)
-			}
-			defer resp.Body.Close()
-			if resp.ContentLength > 0 {
-				log.Printf("[bookreview]从镜像下载数据库%d字节...", resp.ContentLength)
-				data, err := io.ReadAll(resp.Body)
-				if err == nil && len(data) > 0 {
-					_, _ = f.Write(data)
-				} else {
-					panic(err)
-				}
-			}
-		}
 		err := db.Create("bookreview", &book{})
 		if err != nil {
 			panic(err)

plugin_diana/data/text.go

@@ -4,9 +4,7 @@ package data
 import (
 	"crypto/md5"
 	"errors"
-	"io"
 	"math/rand"
-	"net/http"
 	"os"
 	"sync"
 
@@ -21,7 +19,6 @@ import (
 const (
 	datapath = "data/Diana"
 	pbfile   = datapath + "/text.pb"
-	pburl    = "https://codechina.csdn.net/u011570312/ZeroBot-Plugin/-/raw/master/" + pbfile
 )
 
 var (
@@ -56,42 +53,12 @@ func init() {
 
 // LoadText 加载小作文
 func LoadText() error {
-	if file.IsExist(pbfile) {
+	data, err := file.GetLazyData(pbfile, true, false)
-		f, err := os.Open(pbfile)
-		if err == nil {
-			defer f.Close()
-			data, err1 := io.ReadAll(f)
-			if err1 == nil {
-				if len(data) > 0 {
-					return proto.Unmarshal(data, &compo)
-				}
-			}
-			return err1
-		}
-	} else { // 如果没有小作文，则从 url 下载
-		f, err := os.Create(pbfile)
 	if err != nil {
 		return err
 	}
-		defer f.Close()
-		resp, err := http.Get(pburl)
-		if err == nil {
-			defer resp.Body.Close()
-			if resp.ContentLength > 0 {
-				log.Printf("[Diana]从镜像下载小作文%d字节...", resp.ContentLength)
-				data, err := io.ReadAll(resp.Body)
-				if err == nil && len(data) > 0 {
-					_, _ = f.Write(data)
 	return proto.Unmarshal(data, &compo)
 }
-				return err
-			}
-			return nil
-		}
-		return err
-	}
-	return nil
-}
 
 // AddText 添加小作文
 func AddText(txt string) error {

plugin_omikuji/data.go

@@ -1,8 +1,6 @@
 package omikuji
 
 import (
-	"io"
-	"net/http"
 	"os"
 
 	log "github.com/sirupsen/logrus"
@@ -15,7 +13,6 @@ import (
 const (
 	dbpath = "data/Omikuji/"
 	dbfile = dbpath + "kuji.db"
-	dburl  = "https://codechina.csdn.net/u011570312/ZeroBot-Plugin/-/raw/master/" + dbfile
 )
 
 var db = &sql.Sqlite{DBPath: dbfile}
@@ -29,28 +26,7 @@ func init() {
 		}()
 		process.SleepAbout1sTo2s()
 		_ = os.MkdirAll(dbpath, 0755)
-		if !file.IsExist(dbfile) { // 如果没有数据库，则从 url 下载
+		_, _ = file.GetLazyData(dbfile, false, true)
-			f, err := os.Create(dbfile)
-			if err != nil {
-				panic(err)
-			}
-			defer f.Close()
-			resp, err := http.Get(dburl)
-
-			if err != nil {
-				panic(err)
-			}
-			defer resp.Body.Close()
-			if resp.ContentLength > 0 {
-				log.Printf("[omikuji]从镜像下载数据库%d字节...", resp.ContentLength)
-				data, err := io.ReadAll(resp.Body)
-				if err == nil && len(data) > 0 {
-					_, _ = f.Write(data)
-				} else {
-					panic(err)
-				}
-			}
-		}
 		err := db.Create("kuji", &kuji{})
 		if err != nil {
 			panic(err)

plugin_reborn/load.go

@@ -2,8 +2,6 @@ package reborn
 
 import (
 	"encoding/json"
-	"io"
-	"net/http"
 	"os"
 
 	wr "github.com/mroth/weightedrand"
@@ -16,7 +14,6 @@ import (
 const (
 	datapath = "data/Reborn"
 	jsonfile = datapath + "/rate.json"
-	pburl    = "https://codechina.csdn.net/u011570312/ZeroBot-Plugin/-/raw/master/" + jsonfile
 )
 
 type rate []struct {
@@ -55,39 +52,9 @@ func init() {
 
 // load 加载rate数据
 func load(area *rate) error {
-	if file.IsExist(jsonfile) {
+	data, err := file.GetLazyData(jsonfile, true, true)
-		f, err := os.Open(jsonfile)
-		if err == nil {
-			defer f.Close()
-			data, err1 := io.ReadAll(f)
-			if err1 == nil {
-				if len(data) > 0 {
-					return json.Unmarshal(data, area)
-				}
-			}
-			return err1
-		}
-	} else { // 如果没有小作文，则从 url 下载
-		f, err := os.Create(jsonfile)
 	if err != nil {
 		return err
 	}
-		defer f.Close()
-		resp, err := http.Get(pburl)
-		if err == nil {
-			defer resp.Body.Close()
-			if resp.ContentLength > 0 {
-				log.Printf("[Reborn]从镜像下载国家和地区%d字节...", resp.ContentLength)
-				data, err := io.ReadAll(resp.Body)
-				if err == nil && len(data) > 0 {
-					_, _ = f.Write(data)
 	return json.Unmarshal(data, area)
 }
-				return err
-			}
-			return nil
-		}
-		return err
-	}
-	return nil
-}

plugin_setutime/setu_geter.go

@@ -3,8 +3,6 @@ package setutime
 
 import (
 	"fmt"
-	"io"
-	"net/http"
 	"os"
 	"strconv"
 	"strings"
@@ -12,7 +10,6 @@ import (
 	"time"
 
 	"github.com/FloatTech/AnimeAPI/pixiv"
-	"github.com/sirupsen/logrus"
 	zero "github.com/wdvxdr1123/ZeroBot"
 	"github.com/wdvxdr1123/ZeroBot/extension/rate"
 	"github.com/wdvxdr1123/ZeroBot/message"
@@ -58,39 +55,7 @@ func newPools() *imgpool {
 		panic(err)
 	}
 	// 如果数据库不存在则下载
-	if _, err := os.Stat(cache.DB.DBPath); err != nil || os.IsNotExist(err) {
+	_, _ = fileutil.GetLazyData(cache.DB.DBPath, false, false)
-		down := func() (err error) {
-			// 下载
-			resp, err := http.Get(dburl)
-			if err != nil {
-				return
-			}
-			defer resp.Body.Close()
-			if resp.ContentLength > 0 {
-				return
-			}
-			logrus.Printf("[Setu]从镜像下载数据库%d字节...", resp.ContentLength)
-			// 生成文件
-			f, err := os.Create(cache.DB.DBPath)
-			if err != nil {
-				return
-			}
-			defer f.Close()
-			// 读取数据
-			data, err := io.ReadAll(resp.Body)
-			if err != nil || len(data) > 0 {
-				return
-			}
-			// 写入数据
-			if _, err = f.Write(data); err != nil {
-				return
-			}
-			return nil
-		}
-		if err := down(); err != nil {
-			logrus.Printf("[Setu]下载数据库失败%v", err)
-		}
-	}
 	for i := range cache.List {
 		if err := cache.DB.Create(cache.List[i], &pixiv.Illust{}); err != nil {
 			panic(err)

plugin_vtb_quotation/data.go

@@ -1,43 +1,17 @@
 package vtbquotation
 
 import (
-	"io"
-	"net/http"
 	"os"
 
-	log "github.com/sirupsen/logrus"
-
 	"github.com/FloatTech/ZeroBot-Plugin/utils/file"
 	"github.com/FloatTech/ZeroBot-Plugin/utils/process"
 )
 
-const pburl = "https://codechina.csdn.net/u011570312/ZeroBot-Plugin/-/raw/master/" + dbfile
-
 // 加载数据库
 func init() {
 	go func() {
 		process.SleepAbout1sTo2s()
 		_ = os.MkdirAll(dbpath, 0755)
-		if !file.IsExist(dbfile) { // 如果没有数据库，则从 url 下载
+		_, _ = file.GetLazyData(dbfile, false, true)
-			f, err := os.Create(dbfile)
-			if err != nil {
-				panic(err)
-			}
-			defer f.Close()
-			resp, err := http.Get(pburl)
-			if err == nil {
-				defer resp.Body.Close()
-				if resp.ContentLength > 0 {
-					log.Printf("[vtb]从镜像下载数据库%d字节...", resp.ContentLength)
-					data, err := io.ReadAll(resp.Body)
-					if err == nil && len(data) > 0 {
-						_, _ = f.Write(data)
-						return
-					}
-					panic(err)
-				}
-			}
-			panic(err)
-		}
 	}()
 }

utils/file/updater.go

@@ -0,0 +1,101 @@
+package file
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"errors"
+	"io"
+	"net/http"
+	"os"
+	"unsafe"
+
+	reg "github.com/fumiama/go-registry"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	dataurl = "https://codechina.csdn.net/u011570312/ZeroBot-Plugin/-/raw/master/"
+)
+
+var (
+	registry = reg.NewRegReader("reilia.eastasia.azurecontainer.io:32664", "fumiama")
+)
+
+func GetLazyData(path string, isReturnDataBytes, isDataMustEqual bool) ([]byte, error) {
+	var data []byte
+	var resp *http.Response
+	var filemd5 *[16]byte
+	var ms string
+
+	logrus.Infoln("[file]检查懒加载文件:", path)
+	u := dataurl + path
+	err := registry.Connect()
+	if err != nil {
+		logrus.Errorln("[file]无法连接到md5验证服务器，请自行确保下载文件的正确性:", err)
+	} else {
+		ms, err = registry.Get(path)
+		if err != nil || len(ms) != 16 {
+			logrus.Errorln("[file]获取md5失败，请自行确保下载文件的正确性:", err)
+		} else {
+			filemd5 = (*[16]byte)(*(*unsafe.Pointer)(unsafe.Pointer(&ms)))
+			logrus.Infoln("[file]从验证服务器获得文件md5:", hex.EncodeToString(filemd5[:]))
+			_ = registry.Close()
+		}
+	}
+
+	if IsExist(path) {
+		data, err = os.ReadFile(path)
+		if err != nil {
+			return nil, err
+		}
+		if filemd5 != nil {
+			if md5.Sum(data) == *filemd5 {
+				logrus.Infoln("[file]文件md5匹配，文件已存在且为最新")
+				goto ret
+			} else if !isDataMustEqual {
+				logrus.Warnln("[file]文件md5不匹配，但不主动更新")
+				goto ret
+			}
+			logrus.Infoln("[file]文件md5不匹配，开始更新文件")
+		} else {
+			logrus.Warnln("[file]文件存在，已跳过md5检查")
+			goto ret
+		}
+	}
+
+	// 下载
+	resp, err = http.Get(u)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.ContentLength <= 0 {
+		return nil, errors.New("resp body len <= 0")
+	}
+	logrus.Printf("[file]从镜像下载数据%d字节...", resp.ContentLength)
+	// 读取数据
+	data, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if len(data) <= 0 {
+		return nil, errors.New("read body len <= 0")
+	}
+	if filemd5 != nil {
+		if md5.Sum(data) == *filemd5 {
+			logrus.Infoln("[file]文件下载完成，md5匹配，开始保存")
+		} else {
+			logrus.Errorln("[file]文件md5不匹配，下载失败")
+			return nil, errors.New("file md5 mismatch")
+		}
+	} else {
+		logrus.Warnln("[file]文件下载完成，已跳过md5检查，开始保存")
+	}
+	// 写入数据
+	err = os.WriteFile(path, data, 0644)
+ret:
+	if isReturnDataBytes {
+		return data, err
+	}
+	return nil, err
+}