Northern_Lights/NapCatQQ
1
0
Fork 0
mirror of https://github.com/NapNeko/NapCatQQ.git synced 2026-01-17 13:50:36 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

refactor: optimize msg db

Browse Source
This commit is contained in:
linyuchen 2024-04-29 15:17:24 +08:00
parent 43b780cbe6
commit 01417be954
16 changed files with 15 additions and 338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

323
src/common/utils/db.ts
View File

@ -1,323 +0,0 @@
import { ElementType, FileElement, PicElement, PttElement, RawMessage, VideoElement } from '../../core/src/entities';
import sqlite3 from 'sqlite3';
import { log, logDebug, logError } from '@/common/utils/log';
type DBMsg = {
id: number,
longId: string,
seq: number,
peerUid: string,
msg: string
}
type DBFile = {
name: string; // 文件名
path: string;
url: string;
size: number;
uuid: string;
msgId: string;
elementId: string;
element: PicElement | VideoElement | FileElement | PttElement;
elementType: ElementType.PIC | ElementType.VIDEO | ElementType.FILE | ElementType.PTT;
}
class DBUtilBase {
protected db: sqlite3.Database | undefined;
createConnection(dbPath: string) {
if (this.db) {
return;
}
this.db = new sqlite3.Database(dbPath, sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE, (err) => {
if (err) {
logError('Could not connect to database', err);
return;
}
this.createTable();
});
}
protected createTable() {
throw new Error('Method not implemented.');
}
close() {
this.db?.close();
}
}
class DBUtil extends DBUtilBase {
private msgCache: Map<string, RawMessage> = new Map<string, RawMessage>();
constructor() {
super();
const interval = 1000 * 60 * 10; // 10分钟清理一次缓存
setInterval(() => {
logDebug('清理消息缓存');
this.msgCache.forEach((msg, key) => {
if ((Date.now() - parseInt(msg.msgTime) * 1000) > interval) {
this.msgCache.delete(key);
}
});
}, interval);
}
protected createTable() {
// 消息记录
const createTableSQL = `
CREATE TABLE IF NOT EXISTS msgs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
long_id TEXT NOT NULL UNIQUE,
seq INTEGER NOT NULL,
peer_uid TEXT NOT NULL,
msg TEXT NOT NULL
)`;
this.db!.run(createTableSQL, function (err) {
if (err) {
logError('Could not create table', err);
}
});
// 文件缓存
const createFileTableSQL = `
CREATE TABLE IF NOT EXISTS files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
path TEXT NOT NULL,
url TEXT,
size INTEGER NOT NULL,
uuid TEXT,
elementType INTEGER,
element TEXT NOT NULL,
elementId TEXT NOT NULL,
msgId TEXT NOT NULL
)`;
this.db!.run(createFileTableSQL, function (err) {
if (err) {
logError('Could not create table files', err);
}
});
// 接收到的临时会话消息uid
const createTempUinTableSQL = `
CREATE TABLE IF NOT EXISTS temp_uins (
id INTEGER PRIMARY KEY AUTOINCREMENT,
uid TEXT,
uin TEXT
)`;
this.db!.run(createTempUinTableSQL, function (err) {
if (err) {
logError('Could not create table temp_uins', err);
}
});
}
private async getMsg(query: string, params: any[]) {
const stmt = this.db!.prepare(query);
return new Promise<RawMessage | null>((resolve, reject) => {
stmt.get(...params, (err: any, row: DBMsg) => {
// log("getMsg", row, err);
if (err) {
logError('Could not get msg by short id', err);
resolve(null);
}
try {
const msg = JSON.parse(row.msg);
msg.id = row.id;
return resolve(msg);
} catch (e) {
return resolve(null);
}
});
});
}
async getMsgByShortId(shortId: number): Promise<RawMessage | null> {
const getStmt = 'SELECT * FROM msgs WHERE id = ?';
return this.getMsg(getStmt, [shortId]);
}
async getMsgByLongId(longId: string): Promise<RawMessage | null> {
if (this.msgCache.has(longId)) {
return this.msgCache.get(longId)!;
}
return this.getMsg('SELECT * FROM msgs WHERE long_id = ?', [longId]);
}
async getMsgBySeq(peerUid: string, seq: string): Promise<RawMessage | null> {
const stmt = 'SELECT * FROM msgs WHERE peer_uid = ? AND seq = ?';
return this.getMsg(stmt, [peerUid, seq]);
}
async addMsg(msg: RawMessage, update = true): Promise<number> {
logDebug('正在记录消息到数据库', msg.msgId);
const existMsg = await this.getMsgByLongId(msg.msgId);
if (existMsg) {
// logDebug('消息已存在，更新数据库', msg.msgId);
if (update) this.updateMsg(msg).then();
return existMsg.id!;
}
const stmt = this.db!.prepare('INSERT INTO msgs (long_id, seq, peer_uid, msg) VALUES (?, ?, ?, ?)');
// const runAsync = promisify(stmt.run.bind(stmt));
return new Promise((resolve, reject) => {
// eslint-disable-next-line @typescript-eslint/no-this-alias
const dbInstance = this;
stmt.run(msg.msgId, msg.msgSeq, msg.peerUid, JSON.stringify(msg), function (err: any) {
if (err) {
if (err.errno === 19) {
// logDebug('消息已存在，更新数据库', msg.msgId);
dbInstance.getMsgByLongId(msg.msgId).then((msg: RawMessage | null) => {
if (msg) {
dbInstance.msgCache.set(msg.msgId, msg);
// logDebug('获取消息短id成功', msg.id);
resolve(msg.id!);
} else {
logError('db could not get msg by long id', err);
resolve(-1);
}
});
} else {
logError('db could not add msg', err);
resolve(-1);
}
} else {
// log("addMsg", this);
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-expect-error
msg.id = this.lastID;
dbInstance.msgCache.set(msg.msgId, msg);
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// log('获取消息短id成功', this.lastID);
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-expect-error
resolve(this.lastID);
}
});
});
}
async updateMsg(msg: RawMessage) {
const existMsg = this.msgCache.get(msg.msgId);
if (existMsg) {
Object.assign(existMsg, msg);
}
const stmt = this.db!.prepare('UPDATE msgs SET msg = ?, seq = ? WHERE long_id = ?');
try {
stmt.run(JSON.stringify(msg), msg.msgSeq, msg.msgId);
} catch (e) {
logError('updateMsg db error', e);
}
}
async addFileCache(file: DBFile) {
const stmt = this.db!.prepare('INSERT INTO files (name, path, url, size, uuid, elementType ,element, elementId, msgId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');
return new Promise((resolve, reject) => {
stmt.run(file.name, file.path, file.url, file.size, file.uuid,
file.elementType,
JSON.stringify(file.element),
file.elementId,
file.msgId,
function (err: any) {
if (err) {
logError('db could not add file', err);
reject(err);
}
resolve(null);
});
});
}
private async getFileCache(query: string, params: any[]) {
const stmt = this.db!.prepare(query);
return new Promise<DBFile | null>((resolve, reject) => {
stmt.get(...params, (err: any, row: DBFile & { element: string }) => {
if (err) {
logError('db could not get file cache', err);
reject(err);
}
if (row) {
row.element = JSON.parse(row.element);
}
resolve(row);
});
});
}
async getFileCacheByName(name: string): Promise<DBFile | null> {
return this.getFileCache('SELECT * FROM files WHERE name = ?', [name]);
}
async getFileCacheByUuid(uuid: string): Promise<DBFile | null> {
return this.getFileCache('SELECT * FROM files WHERE uuid = ?', [uuid]);
}
// todo: 是否所有的文件都有uuid？语音消息有没有uuid？
async updateFileCache(file: DBFile) {
const stmt = this.db!.prepare('UPDATE files SET path = ?, url = ? WHERE uuid = ?');
return new Promise((resolve, reject) => {
stmt.run(file.path, file.url,file.uuid, function (err: any) {
if (err) {
logError('db could not update file cache', err);
reject(err);
}
resolve(null);
});
});
}
// 被动收到的临时会话消息uin->uid
async getReceivedTempUinMap() {
const stmt = 'SELECT * FROM temp_uins';
return new Promise<Record<string, string>>((resolve, reject) => {
this.db!.all(stmt, (err, rows: { uin: string, uid: string }[]) => {
if (err) {
logError('db could not get temp uin map', err);
reject(err);
}
const map: Record<string, string> = {};
rows.forEach(row => {
map[row.uin] = row.uid;
});
resolve(map);
});
});
}
// 通过uin获取临时会话消息uid
async getUidByTempUin(uid: string) {
const stmt = 'SELECT * FROM temp_uins WHERE uin = ?';
return new Promise<string>((resolve, reject) => {
this.db!.get(stmt, [uid], (err, row: { uin: string, uid: string }) => {
if (err) {
logError('db could not get temp uin map', err);
reject(err);
}
resolve(row?.uid);
});
});
}
async addTempUin(uin: string, uid: string) {
const existUid = await this.getUidByTempUin(uin);
if (!existUid) {
const stmt = this.db!.prepare('INSERT INTO temp_uins (uin, uid) VALUES (?, ?)');
return new Promise((resolve, reject) => {
stmt.run(uin, uid, function (err: any) {
if (err) {
logError('db could not add temp uin', err);
reject(err);
}
resolve(null);
});
});
}
}
}
export const dbUtil = new DBUtil();

2
src/common/utils/file.ts
View File

@ -4,7 +4,7 @@ import crypto from 'crypto';
import util from 'util'; import util from 'util';
import path from 'node:path'; import path from 'node:path';
import { log } from './log'; import { log } from './log';
import { dbUtil } from './db'; import { dbUtil } from '@/core/utils/db';
import * as fileType from 'file-type'; import * as fileType from 'file-type';
import { v4 as uuidv4 } from 'uuid'; import { v4 as uuidv4 } from 'uuid';
import { napCatCore } from '@/core'; import { napCatCore } from '@/core';

2
src/onebot11/action/extends/Debug.ts
View File

@ -26,7 +26,7 @@ export default class Debug extends BaseAction<Payload, any> {
// NTQQFileCacheApi, // NTQQFileCacheApi,
NTQQWindowApi]; NTQQWindowApi];
for (const ntqqApiClass of ntqqApi) { for (const ntqqApiClass of ntqqApi) {
logDebug('ntqqApiClass', ntqqApiClass); // logDebug('ntqqApiClass', ntqqApiClass);
const method = (<any>ntqqApiClass)[payload.method]; const method = (<any>ntqqApiClass)[payload.method];
if (method) { if (method) {
const result = method(...payload.args); const result = method(...payload.args);

2
src/onebot11/action/file/GetFile.ts
View File

@ -1,6 +1,6 @@
import BaseAction from '../BaseAction'; import BaseAction from '../BaseAction';
import fs from 'fs/promises'; import fs from 'fs/promises';
import { dbUtil } from '@/common/utils/db'; import { dbUtil } from '@/core/utils/db';
import { ob11Config } from '@/onebot11/config'; import { ob11Config } from '@/onebot11/config';
import { log, logDebug } from '@/common/utils/log'; import { log, logDebug } from '@/common/utils/log';
import { sleep } from '@/common/utils/helper'; import { sleep } from '@/common/utils/helper';

2
src/onebot11/action/go-cqhttp/GetForwardMsg.ts
View File

@ -1,7 +1,7 @@
import BaseAction from '../BaseAction'; import BaseAction from '../BaseAction';
import { OB11ForwardMessage, OB11Message, OB11MessageData } from '../../types'; import { OB11ForwardMessage, OB11Message, OB11MessageData } from '../../types';
import { NTQQMsgApi } from '@/core/apis'; import { NTQQMsgApi } from '@/core/apis';
import { dbUtil } from '@/common/utils/db'; import { dbUtil } from '@/core/utils/db';
import { OB11Constructor } from '../../constructor'; import { OB11Constructor } from '../../constructor';
import { ActionName } from '../types'; import { ActionName } from '../types';

2
src/onebot11/action/go-cqhttp/GetFriendMsgHistory.ts
View File

@ -3,7 +3,7 @@ import { OB11Message, OB11User } from '../../types';
import { getFriend, friends, uid2UinMap, getUidByUin } from '@/core/data'; import { getFriend, friends, uid2UinMap, getUidByUin } from '@/core/data';
import { ActionName } from '../types'; import { ActionName } from '../types';
import { ChatType } from '@/core/entities'; import { ChatType } from '@/core/entities';
import { dbUtil } from '@/common/utils/db'; import { dbUtil } from '@/core/utils/db';
import { NTQQMsgApi } from '@/core/apis/msg'; import { NTQQMsgApi } from '@/core/apis/msg';
import { OB11Constructor } from '../../constructor'; import { OB11Constructor } from '../../constructor';
import { logDebug } from '@/common/utils/log'; import { logDebug } from '@/common/utils/log';

2
src/onebot11/action/go-cqhttp/GetGroupMsgHistory.ts
View File

@ -3,7 +3,7 @@ import { OB11Message, OB11User } from '../../types';
import { getGroup, groups } from '@/core/data'; import { getGroup, groups } from '@/core/data';
import { ActionName } from '../types'; import { ActionName } from '../types';
import { ChatType } from '@/core/entities'; import { ChatType } from '@/core/entities';
import { dbUtil } from '@/common/utils/db'; import { dbUtil } from '@/core/utils/db';
import { NTQQMsgApi } from '@/core/apis/msg'; import { NTQQMsgApi } from '@/core/apis/msg';
import { OB11Constructor } from '../../constructor'; import { OB11Constructor } from '../../constructor';
import { logDebug } from '@/common/utils/log'; import { logDebug } from '@/common/utils/log';

2
src/onebot11/action/msg/DeleteMsg.ts
View File

@ -1,7 +1,7 @@
import { NTQQMsgApi } from '@/core/apis'; import { NTQQMsgApi } from '@/core/apis';
import { ActionName } from '../types'; import { ActionName } from '../types';
import BaseAction from '../BaseAction'; import BaseAction from '../BaseAction';
import { dbUtil } from '@/common/utils/db'; import { dbUtil } from '@/core/utils/db';
import { napCatCore } from '@/core'; import { napCatCore } from '@/core';
interface Payload { interface Payload {

2
src/onebot11/action/msg/GetMsg.ts
View File

@ -2,7 +2,7 @@ import { OB11Message } from '../../types';
import { OB11Constructor } from '../../constructor'; import { OB11Constructor } from '../../constructor';
import BaseAction from '../BaseAction'; import BaseAction from '../BaseAction';
import { ActionName } from '../types'; import { ActionName } from '../types';
import { dbUtil } from '@/common/utils/db'; import { dbUtil } from '@/core/utils/db';
export interface PayloadType { export interface PayloadType {

2
src/onebot11/action/msg/SendMsg.ts
View File

@ -22,7 +22,7 @@ import BaseAction from '../BaseAction';
import { ActionName, BaseCheckResult } from '../types'; import { ActionName, BaseCheckResult } from '../types';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { decodeCQCode } from '../../cqcode'; import { decodeCQCode } from '../../cqcode';
import { dbUtil } from '@/common/utils/db'; import { dbUtil } from '@/core/utils/db';
import { log, logDebug, logError } from '@/common/utils/log'; import { log, logDebug, logError } from '@/common/utils/log';
import { sleep } from '@/common/utils/helper'; import { sleep } from '@/common/utils/helper';
import { uri2local } from '@/common/utils/file'; import { uri2local } from '@/common/utils/file';

2
src/onebot11/action/msg/SetMsgEmojiLike.ts
View File

@ -1,6 +1,6 @@
import { ActionName } from '../types'; import { ActionName } from '../types';
import BaseAction from '../BaseAction'; import BaseAction from '../BaseAction';
import { dbUtil } from '@/common/utils/db'; import { dbUtil } from '@/core/utils/db';
import { NTQQMsgApi } from '@/core/apis'; import { NTQQMsgApi } from '@/core/apis';
interface Payload { interface Payload {

2
src/onebot11/action/system/CleanCache.ts
View File

@ -7,7 +7,7 @@ import {
ChatCacheListItemBasic, ChatCacheListItemBasic,
CacheFileType CacheFileType
} from '@/core/entities'; } from '@/core/entities';
import { dbUtil } from '../../../common/utils/db'; import { dbUtil } from '@/core/utils/db';
import { NTQQFileApi, NTQQFileCacheApi } from '@/core/apis/file'; import { NTQQFileApi, NTQQFileCacheApi } from '@/core/apis/file';
export default class CleanCache extends BaseAction<void, void> { export default class CleanCache extends BaseAction<void, void> {

2
src/onebot11/action/types.ts
View File

@ -19,7 +19,7 @@ export enum ActionName {
SetQQAvatar = 'set_qq_avatar', SetQQAvatar = 'set_qq_avatar',
GetConfig = 'get_config', GetConfig = 'get_config',
SetConfig = 'set_config', SetConfig = 'set_config',
Debug = 'llonebot_debug', Debug = 'debug',
GetFile = 'get_file', GetFile = 'get_file',
// onebot 11 // onebot 11
SendLike = 'send_like', SendLike = 'send_like',

2
src/onebot11/constructor.ts
View File

@ -26,7 +26,7 @@ import {
} from '../core/src/entities'; } from '../core/src/entities';
import { EventType } from './event/OB11BaseEvent'; import { EventType } from './event/OB11BaseEvent';
import { encodeCQCode } from './cqcode'; import { encodeCQCode } from './cqcode';
import { dbUtil } from '@/common/utils/db'; import { dbUtil } from '@/core/utils/db';
import { OB11GroupIncreaseEvent } from './event/notice/OB11GroupIncreaseEvent'; import { OB11GroupIncreaseEvent } from './event/notice/OB11GroupIncreaseEvent';
import { OB11GroupBanEvent } from './event/notice/OB11GroupBanEvent'; import { OB11GroupBanEvent } from './event/notice/OB11GroupBanEvent';
import { OB11GroupUploadNoticeEvent } from './event/notice/OB11GroupUploadNoticeEvent'; import { OB11GroupUploadNoticeEvent } from './event/notice/OB11GroupUploadNoticeEvent';

2
src/onebot11/main.ts
View File

@ -16,7 +16,7 @@ import { httpHeart, ob11HTTPServer } from '@/onebot11/server/http';
import { ob11WebsocketServer } from '@/onebot11/server/ws/WebsocketServer'; import { ob11WebsocketServer } from '@/onebot11/server/ws/WebsocketServer';
import { ob11ReverseWebsockets } from '@/onebot11/server/ws/ReverseWebsocket'; import { ob11ReverseWebsockets } from '@/onebot11/server/ws/ReverseWebsocket';
import { friendRequests, getFriend, getGroup, getGroupMember, groupNotifies, selfInfo } from '@/core/data'; import { friendRequests, getFriend, getGroup, getGroupMember, groupNotifies, selfInfo } from '@/core/data';
import { dbUtil } from '@/common/utils/db'; import { dbUtil } from '@/core/utils/db';
import { BuddyListener, GroupListener, NodeIKernelBuddyListener } from '../core/src/listeners'; import { BuddyListener, GroupListener, NodeIKernelBuddyListener } from '../core/src/listeners';
import { OB11FriendRequestEvent } from '@/onebot11/event/request/OB11FriendRequest'; import { OB11FriendRequestEvent } from '@/onebot11/event/request/OB11FriendRequest';
import { NTQQGroupApi, NTQQUserApi } from '../core/src/apis'; import { NTQQGroupApi, NTQQUserApi } from '../core/src/apis';

2
src/onebot11/server/postOB11Event.ts
View File

@ -11,7 +11,7 @@ import { convertMessage2List, createSendElements, sendMsg } from '../action/msg/
import { OB11FriendRequestEvent } from '../event/request/OB11FriendRequest'; import { OB11FriendRequestEvent } from '../event/request/OB11FriendRequest';
import { OB11GroupRequestEvent } from '../event/request/OB11GroupRequest'; import { OB11GroupRequestEvent } from '../event/request/OB11GroupRequest';
import { isNull } from '@/common/utils/helper'; import { isNull } from '@/common/utils/helper';
import { dbUtil } from '@/common/utils/db'; import { dbUtil } from '@/core/utils/db';
import { friendRequests, getGroup, groupNotifies, selfInfo } from '@/core/data'; import { friendRequests, getGroup, groupNotifies, selfInfo } from '@/core/data';
import { NTQQFriendApi, NTQQGroupApi, NTQQMsgApi } from '../../core/src/apis'; import { NTQQFriendApi, NTQQGroupApi, NTQQMsgApi } from '../../core/src/apis';