🎨 ⚡️ 🔥 优化 jandan

data

@@ -1 +1 @@
-Subproject commit 663410335c3a6a340bc9c0df0922c08aa5037edd
+Subproject commit 0090d93bdad741aa5b1df8e18e5a5ed7ea414fdc

plugin_jandan/cron.go

@@ -1,74 +0,0 @@
-package jandan
-
-import (
-	"github.com/FloatTech/zbputils/process"
-	"github.com/antchfx/htmlquery"
-	"github.com/fumiama/cron"
-	log "github.com/sirupsen/logrus"
-	"regexp"
-	"strconv"
-)
-
-var (
-	chanPicture = make(chan string, 100000)
-	pageTotal   int
-)
-
-func init() {
-	go func() {
-		process.SleepAbout1sTo2s()
-		scorePicture()
-		log.Println("[jandan/cron] 开启jandan数据库日常更新")
-		jandanDaily()
-	}()
-	travelWebpage()
-}
-
-func jandanDaily() {
-	c := cron.New()
-	_, err := c.AddFunc("10 4 * * *", func() { travelWebpage() })
-	if err != nil {
-		log.Errorln("定时任务有错误:", err)
-	} else {
-		log.Println("开启jandan数据库定时任务")
-		c.Start()
-	}
-}
-
-func travelWebpage() {
-	pictureList = pictureList[0:0]
-	webpageURL := jandanPictureURL
-	doc, err := htmlquery.LoadURL(webpageURL)
-	if err != nil {
-		log.Errorln("[jandan]:", err)
-	}
-	re := regexp.MustCompile(`\d+`)
-	pageTotal, err = strconv.Atoi(re.FindString(htmlquery.FindOne(doc, "//*[@id='comments']/div[2]/div/span[@class='current-comment-page']/text()").Data))
-	if err != nil {
-		log.Errorln("[jandan]:", err)
-	}
-	for i := 0; i < pageTotal; i++ {
-		doc, err = htmlquery.LoadURL(webpageURL)
-		if err != nil {
-			log.Errorln("[jandan]:", err)
-		}
-		picList, err := htmlquery.QueryAll(doc, "//*[@class='view_img_link']")
-		if err != nil {
-			log.Errorln("[jandan]:", err)
-		}
-		if len(picList) != 0 {
-			for _, v := range picList {
-				chanPicture <- "https:" + v.Attr[0].Val
-			}
-		}
-		if i != pageTotal-1 {
-			webpageURL = "https:" + htmlquery.FindOne(doc, "//*[@id='comments']/div[@class='comments']/div[@class='cp-pagenavi']/a[@class='previous-comment-page']").Attr[1].Val
-		}
-	}
-}
-
-func scorePicture() {
-	for pictureURL := range chanPicture {
-		pictureList = append(pictureList, pictureURL)
-	}
-}

plugin_jandan/data.go

@@ -0,0 +1,51 @@
+package jandan
+
+import (
+	"os"
+	"sync"
+
+	"github.com/FloatTech/ZeroBot-Plugin/order"
+	"github.com/FloatTech/zbputils/file"
+	"github.com/FloatTech/zbputils/process"
+	"github.com/FloatTech/zbputils/sql"
+	log "github.com/sirupsen/logrus"
+)
+
+const dbpath = "data/Jandan/"
+const dbfile = dbpath + "pics.db"
+
+var db = &sql.Sqlite{DBPath: dbfile}
+var mu sync.RWMutex
+
+type picture struct {
+	ID  uint64 `db:"id"`
+	URL string `db:"url"`
+}
+
+func getRandomPicture() (u string, err error) {
+	var p picture
+	mu.RLock()
+	err = db.Pick("picture", &p)
+	mu.RUnlock()
+	u = p.URL
+	return
+}
+
+// 加载数据库
+func init() {
+	go func() {
+		defer order.DoneOnExit()()
+		process.SleepAbout1sTo2s()
+		_ = os.MkdirAll(dbpath, 0755)
+		_, _ = file.GetLazyData(dbfile, false, false)
+		err := db.Create("picture", &picture{})
+		if err != nil {
+			panic(err)
+		}
+		n, err := db.Count("picture")
+		if err != nil {
+			panic(err)
+		}
+		log.Printf("[jandan]读取%d张图片", n)
+	}()
+}

plugin_jandan/jandan.go

@@ -2,21 +2,23 @@
 package jandan
 
 import (
-	"github.com/FloatTech/ZeroBot-Plugin/order"
+	"fmt"
-	"github.com/FloatTech/zbputils/control"
+	"hash/crc64"
-	"github.com/wdvxdr1123/ZeroBot/message"
+	"regexp"
-	"math/rand"
+	"strconv"
-	"time"
 
+	"github.com/FloatTech/zbputils/binary"
+	"github.com/FloatTech/zbputils/control"
+	"github.com/antchfx/htmlquery"
+	"github.com/sirupsen/logrus"
 	zero "github.com/wdvxdr1123/ZeroBot"
+	"github.com/wdvxdr1123/ZeroBot/message"
+
+	"github.com/FloatTech/ZeroBot-Plugin/order"
 )
 
 const (
-	jandanPictureURL = "http://jandan.net/pic"
+	api = "http://jandan.net/pic"
-)
-
-var (
-	pictureList []string
 )
 
 func init() {
@@ -27,13 +29,63 @@ func init() {
 
 	engine.OnFullMatch("来份屌图").SetBlock(true).
 		Handle(func(ctx *zero.Ctx) {
-			rand.Seed(time.Now().Unix())
+			u, err := getRandomPicture()
-			ctx.SendChain(message.Image(pictureList[rand.Intn(len(pictureList))]))
+			if err != nil {
+				ctx.SendChain(message.Text("ERROR:", err))
+				return
+			}
+			ctx.SendChain(message.Image(u))
 		})
 
 	engine.OnFullMatch("更新屌图", zero.SuperUserPermission).SetBlock(true).
 		Handle(func(ctx *zero.Ctx) {
-			ctx.Send("少女更新中......")
+			ctx.Send("少女更新中...")
-			travelWebpage()
+			webpageURL := api
+			doc, err := htmlquery.LoadURL(webpageURL)
+			if err != nil {
+				ctx.SendChain(message.Text("ERROR:", err))
+				return
+			}
+			re := regexp.MustCompile(`\d+`)
+			pageTotal, err := strconv.Atoi(re.FindString(htmlquery.FindOne(doc, "//*[@id='comments']/div[2]/div/span[@class='current-comment-page']/text()").Data))
+			if err != nil {
+				ctx.SendChain(message.Text("ERROR:", err))
+				return
+			}
+		LOOP:
+			for i := 0; i < pageTotal; i++ {
+				logrus.Infoln("[jandan]", fmt.Sprintf("处理第%d/%d页...", i, pageTotal))
+				doc, err = htmlquery.LoadURL(webpageURL)
+				if err != nil {
+					ctx.SendChain(message.Text("ERROR:", err))
+					return
+				}
+				picList, err := htmlquery.QueryAll(doc, "//*[@class='view_img_link']")
+				if err != nil {
+					ctx.SendChain(message.Text("ERROR:", err))
+					return
+				}
+				if len(picList) != 0 {
+					for _, v := range picList {
+						u := "https:" + v.Attr[0].Val
+						i := crc64.Checksum(binary.StringToBytes(u), crc64.MakeTable(crc64.ISO))
+						mu.RLock()
+						ok := db.CanFind("picture", "where id="+strconv.FormatUint(i, 10))
+						mu.RUnlock()
+						if !ok {
+							mu.Lock()
+							db.Insert("picture", &picture{ID: i, URL: u})
+							mu.Unlock()
+						} else {
+							// 开始重复，说明之后都是重复
+							break LOOP
+						}
+					}
+				}
+				if i != pageTotal-1 {
+					webpageURL = "https:" + htmlquery.FindOne(doc, "//*[@id='comments']/div[@class='comments']/div[@class='cp-pagenavi']/a[@class='previous-comment-page']").Attr[1].Val
+				}
+			}
+			ctx.Send("更新完成!")
 		})
 }