diff --git a/data b/data index 66341033..0090d93b 160000 --- a/data +++ b/data @@ -1 +1 @@ -Subproject commit 663410335c3a6a340bc9c0df0922c08aa5037edd +Subproject commit 0090d93bdad741aa5b1df8e18e5a5ed7ea414fdc diff --git a/plugin_jandan/cron.go b/plugin_jandan/cron.go deleted file mode 100644 index 9a54bd2d..00000000 --- a/plugin_jandan/cron.go +++ /dev/null @@ -1,74 +0,0 @@ -package jandan - -import ( - "github.com/FloatTech/zbputils/process" - "github.com/antchfx/htmlquery" - "github.com/fumiama/cron" - log "github.com/sirupsen/logrus" - "regexp" - "strconv" -) - -var ( - chanPicture = make(chan string, 100000) - pageTotal int -) - -func init() { - go func() { - process.SleepAbout1sTo2s() - scorePicture() - log.Println("[jandan/cron] 开启jandan数据库日常更新") - jandanDaily() - }() - travelWebpage() -} - -func jandanDaily() { - c := cron.New() - _, err := c.AddFunc("10 4 * * *", func() { travelWebpage() }) - if err != nil { - log.Errorln("定时任务有错误:", err) - } else { - log.Println("开启jandan数据库定时任务") - c.Start() - } -} - -func travelWebpage() { - pictureList = pictureList[0:0] - webpageURL := jandanPictureURL - doc, err := htmlquery.LoadURL(webpageURL) - if err != nil { - log.Errorln("[jandan]:", err) - } - re := regexp.MustCompile(`\d+`) - pageTotal, err = strconv.Atoi(re.FindString(htmlquery.FindOne(doc, "//*[@id='comments']/div[2]/div/span[@class='current-comment-page']/text()").Data)) - if err != nil { - log.Errorln("[jandan]:", err) - } - for i := 0; i < pageTotal; i++ { - doc, err = htmlquery.LoadURL(webpageURL) - if err != nil { - log.Errorln("[jandan]:", err) - } - picList, err := htmlquery.QueryAll(doc, "//*[@class='view_img_link']") - if err != nil { - log.Errorln("[jandan]:", err) - } - if len(picList) != 0 { - for _, v := range picList { - chanPicture <- "https:" + v.Attr[0].Val - } - } - if i != pageTotal-1 { - webpageURL = "https:" + htmlquery.FindOne(doc, "//*[@id='comments']/div[@class='comments']/div[@class='cp-pagenavi']/a[@class='previous-comment-page']").Attr[1].Val - } - } -} - -func scorePicture() { - for pictureURL := range chanPicture { - pictureList = append(pictureList, pictureURL) - } -} diff --git a/plugin_jandan/data.go b/plugin_jandan/data.go new file mode 100644 index 00000000..892600ad --- /dev/null +++ b/plugin_jandan/data.go @@ -0,0 +1,51 @@ +package jandan + +import ( + "os" + "sync" + + "github.com/FloatTech/ZeroBot-Plugin/order" + "github.com/FloatTech/zbputils/file" + "github.com/FloatTech/zbputils/process" + "github.com/FloatTech/zbputils/sql" + log "github.com/sirupsen/logrus" +) + +const dbpath = "data/Jandan/" +const dbfile = dbpath + "pics.db" + +var db = &sql.Sqlite{DBPath: dbfile} +var mu sync.RWMutex + +type picture struct { + ID uint64 `db:"id"` + URL string `db:"url"` +} + +func getRandomPicture() (u string, err error) { + var p picture + mu.RLock() + err = db.Pick("picture", &p) + mu.RUnlock() + u = p.URL + return +} + +// 加载数据库 +func init() { + go func() { + defer order.DoneOnExit()() + process.SleepAbout1sTo2s() + _ = os.MkdirAll(dbpath, 0755) + _, _ = file.GetLazyData(dbfile, false, false) + err := db.Create("picture", &picture{}) + if err != nil { + panic(err) + } + n, err := db.Count("picture") + if err != nil { + panic(err) + } + log.Printf("[jandan]读取%d张图片", n) + }() +} diff --git a/plugin_jandan/jandan.go b/plugin_jandan/jandan.go index ff28929f..cb4234a1 100644 --- a/plugin_jandan/jandan.go +++ b/plugin_jandan/jandan.go @@ -2,21 +2,23 @@ package jandan import ( - "github.com/FloatTech/ZeroBot-Plugin/order" - "github.com/FloatTech/zbputils/control" - "github.com/wdvxdr1123/ZeroBot/message" - "math/rand" - "time" + "fmt" + "hash/crc64" + "regexp" + "strconv" + "github.com/FloatTech/zbputils/binary" + "github.com/FloatTech/zbputils/control" + "github.com/antchfx/htmlquery" + "github.com/sirupsen/logrus" zero "github.com/wdvxdr1123/ZeroBot" + "github.com/wdvxdr1123/ZeroBot/message" + + "github.com/FloatTech/ZeroBot-Plugin/order" ) const ( - jandanPictureURL = "http://jandan.net/pic" -) - -var ( - pictureList []string + api = "http://jandan.net/pic" ) func init() { @@ -27,13 +29,63 @@ func init() { engine.OnFullMatch("来份屌图").SetBlock(true). Handle(func(ctx *zero.Ctx) { - rand.Seed(time.Now().Unix()) - ctx.SendChain(message.Image(pictureList[rand.Intn(len(pictureList))])) + u, err := getRandomPicture() + if err != nil { + ctx.SendChain(message.Text("ERROR:", err)) + return + } + ctx.SendChain(message.Image(u)) }) engine.OnFullMatch("更新屌图", zero.SuperUserPermission).SetBlock(true). Handle(func(ctx *zero.Ctx) { - ctx.Send("少女更新中......") - travelWebpage() + ctx.Send("少女更新中...") + webpageURL := api + doc, err := htmlquery.LoadURL(webpageURL) + if err != nil { + ctx.SendChain(message.Text("ERROR:", err)) + return + } + re := regexp.MustCompile(`\d+`) + pageTotal, err := strconv.Atoi(re.FindString(htmlquery.FindOne(doc, "//*[@id='comments']/div[2]/div/span[@class='current-comment-page']/text()").Data)) + if err != nil { + ctx.SendChain(message.Text("ERROR:", err)) + return + } + LOOP: + for i := 0; i < pageTotal; i++ { + logrus.Infoln("[jandan]", fmt.Sprintf("处理第%d/%d页...", i, pageTotal)) + doc, err = htmlquery.LoadURL(webpageURL) + if err != nil { + ctx.SendChain(message.Text("ERROR:", err)) + return + } + picList, err := htmlquery.QueryAll(doc, "//*[@class='view_img_link']") + if err != nil { + ctx.SendChain(message.Text("ERROR:", err)) + return + } + if len(picList) != 0 { + for _, v := range picList { + u := "https:" + v.Attr[0].Val + i := crc64.Checksum(binary.StringToBytes(u), crc64.MakeTable(crc64.ISO)) + mu.RLock() + ok := db.CanFind("picture", "where id="+strconv.FormatUint(i, 10)) + mu.RUnlock() + if !ok { + mu.Lock() + db.Insert("picture", &picture{ID: i, URL: u}) + mu.Unlock() + } else { + // 开始重复，说明之后都是重复 + break LOOP + } + } + } + if i != pageTotal-1 { + webpageURL = "https:" + htmlquery.FindOne(doc, "//*[@id='comments']/div[@class='comments']/div[@class='cp-pagenavi']/a[@class='previous-comment-page']").Attr[1].Val + } + } + ctx.Send("更新完成!") }) }